I'm comparing the learning curves of Bernoulli and Multinomial Naive Bayes using the 20_newsgroups dataset from scikit-learn for text-classification. I considered both the "training score" and the "cross validation score", but I noticed that while in the Multinomial version the training score is very high at the beginning and decreases and the cross-validation score is very low at the beginning and increases, in the Bernoulli version I have a low training score at the beginning (and then it increases). Is it normal or am I doing something wrong? It sounds a bit strange to me.
Here is some of my Python code (Bernoulli version):
####load dataset#### from sklearn.datasets import fetch_20newsgroups categories = ['alt.atheism', 'sci.electronics','rec.sport.hockey'] train = fetch_20newsgroups(subset='train', categories=categories, shuffle=True, random_state=42) y = train.target test = fetch_20newsgroups(subset='test', categories=categories, shuffle=True, random_state=42) ####bag of words#### from nltk.corpus import stopwords stopwords = stopwords.words('english') from sklearn.feature_extraction.text import CountVectorizer count_vectorizer = CountVectorizer(stop_words=stopwords, binary=True) matrix_train = count_vectorizer.fit_transform(train.data) from sklearn.naive_bayes import BernoulliNB bernoulli = BernoulliNB(alpha = 1.0, fit_prior = True) ####learning curve#### import matplotlib.pyplot as plt from sklearn.learning_curve import learning_curve def plot_learning_curve(estimator, title, X, y, ylim, cv, n_jobs=1, train_sizes=np.linspace(.1, 1.0, 8)): plt.figure() plt.title(title) plt.xlabel("Training examples") plt.ylabel("Score") train_sizes, train_scores, test_scores = learning_curve( estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes) train_scores_mean = np.mean(train_scores, axis=1) train_scores_std = np.std(train_scores, axis=1) test_scores_mean = np.mean(test_scores, axis=1) test_scores_std = np.std(test_scores, axis=1) plt.grid() plt.fill_between(train_sizes, train_scores_mean - train_scores_std, train_scores_mean + train_scores_std, alpha=0.1, color="r") plt.fill_between(train_sizes, test_scores_mean - test_scores_std, test_scores_mean + test_scores_std, alpha=0.1, color="g") plt.plot(train_sizes, train_scores_mean, 'o-', color="r", label="Training score") plt.plot(train_sizes, test_scores_mean, 'o-', color="g", label="Cross-validation score") plt.legend(loc="best") return plt title = "Learning Curves (Naive Bayes)" from sklearn import cross_validation cv = cross_validation.ShuffleSplit(matrix_train.shape, n_iter=100, test_size=0.2, random_state=0) plot_learning_curve(bernoulli, title, matrix_train, y, ylim=(0.7, 1.01), cv=cv, n_jobs=1) plt.show()
Why are they so different? The cross validation score is like what I was expecting both in Multinomial and Bernoulli, but the training score should be high at the beginning, right?