def loadDataSet():
    '''
    从文本中构建词向量
    :return: 文本集，分类标签集
    '''
    postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                   ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                   ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                   ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                   ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                   ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    classVec = [0, 1, 0, 1, 0, 1]  # 1表示侮辱性文字，0表示正常文字
    return postingList, classVec

def createVocabList(dataSet):
    '''
    创建一个包含所有文本中出现的不重复次的词汇列表
    :param dataSet: 文本集
    :return: 不重复的词汇表
    '''
    vocabSet = set([])  # 创建空集
    for document in dataSet:
        vocabSet = vocabSet | set(document)  # 使用set函数的属性来去除重复单词
    return list(vocabSet)

def setOfWords2Vec(vocabList, inputSet):
    '''
    文档向量对比函数
    :param vocabList: 词汇表向量
    :param inputSet: 文档
    :return: 对比处理后的向量结果集
    '''
    returnVec = [0] * len(vocabList)#创建一个与单词表相同长度的向量并都初始化为0
    for word in inputSet:#遍历该文档，若出现单词表中的单词则将输出文档向量对应位置设为1
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else:
            print "the word: %s is not in my Vocabulary!" % word
    return returnVec

def trainNB0(trainMatrix,trainCategory):
    '''
    朴素贝叶斯分类器训练函数
    :param trainMatrix: 文档矩阵
    :param trainCategory: 文档类别标签向量
    :return: 
    '''
    numTrainDocs=len(trainMatrix)
    numWords=len(trainMatrix[0])
    #初始化概率
    pAbusive=sum(trainCategory)/float(numTrainDocs)
    p0Num=ones(numWords)
    p1Num=ones(numWords)
    p0Denom=2.0
    p1Denom=2.0
    for i in range(numTrainDocs):
        if trainCategory[i]==1:
            p1Num+=trainMatrix[i]
            p1Denom+=sum(trainMatrix[i])
        else:
            p0Num += trainMatrix[i]
            p0Denom += sum(trainMatrix[i])
    p0Vect = log(p0Num/p0Denom)
    p1Vect = log(p1Num/p1Denom)

    return p0Vect,p1Vect,pAbusive

def classifyNB(vec2Classify,p0Vec,p1Vec,pClass1):
    '''
    朴素贝叶斯分类器
    :param vec2Classify: 
    :param p0Vec: 
    :param p1Vec: 
    :param pClass1: 
    :return: 
    '''
    p1=sum(vec2Classify*p1Vec)+log(pClass1)
    p0=sum(vec2Classify*p0Vec)+log(1.0-pClass1)
    if p1>p0:
        return 1
    else:
        return 0

def testingNB():
    listOPosts, listClasses = loadDataSet()
    myVocabList = createVocabList(listOPosts)
    trainMat = []
    for postinDoc in listOPosts:  # 分割输入文本为文本向量
        trainMat.append(setOfWords2Vec(myVocabList, postinDoc))
    p0v, p1v, pAb = trainNB0(array(trainMat),array(listClasses))
    testEntry = ['love', 'my','yes']
    thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
    print testEntry, 'classified as: ', classifyNB(thisDoc, p0v, p1v, pAb)
    testEntry=['stupid','garbage']
    thisDoc=array(setOfWords2Vec(myVocabList,testEntry))
    print testEntry,'classified as: ',classifyNB(thisDoc,p0v,p1v,pAb)

def textParse(bigString):
    '''
    文本处理函数
    :param bigString: 文本
    :return: 处理好的单词
    '''
    import re
    #使用正则表达式来切分句子，分隔符是除单词、数字外的任意字符串
    listOfTokens=re.split(r'\W*',bigString)
    #返回时要注意去掉空字符串而且对于该项目来说我们希望所有词的形式都是统一的因此将所有字符变为小写
    return [tok.lower() for tok in listOfTokens if len(tok)>2]

def spamTest():
    '''
    自动化处理函数，用于测试算法，使用朴素贝叶斯进行交叉验证
    :return: 
    '''
    docList=[]
    classList=[]
    fullText=[]
    for i in range(1,26):
        wordList=textParse(open("email/spam/%d.txt" %i).read())
        #注意两种添加元素的方法是不同的
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(1)
        wordList=textParse(open("email/ham/%d.txt" %i).read())
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(0)
    vocabList=createVocabList(docList)
    #训练集
    trainingSet=range(50)
    #测试集
    testSet=[]
    '''
    随机抽取10个文件，选择10个文件到测试集，并将其从训练集中剔除，
    这种随机选择数据的一部分作为训练集，而剩余部分作为测试集的过程称为存留交叉验证
    '''
    for i in range(10):
        randIndex=int(random.uniform(0,len(trainingSet)))
        testSet.append(trainingSet[randIndex])
        del(trainingSet[randIndex])
    trainMat=[]
    trainClasses=[]

    #将选好的10个样本存入训练矩阵和训练分类器中
    for docIndex in trainingSet:
        trainMat.append(setOfWords2Vec(vocabList,docList[docIndex]))
        trainClasses.append(classList[docIndex])

    p0v,p1v,pSpam=trainNB0(array(trainMat),array(trainClasses))
    errorCount=0
    #对测试集分类
    for docIndex in testSet:
        wordVector=setOfWords2Vec(vocabList,docList[docIndex])
        if classifyNB(array(wordVector),p0v,p1v,pSpam)!=classList[docIndex]:
            errorCount+=1
    print 'the error rate is ',float(errorCount)/len(testSet)