数据分析实例：MovieLens电影数据分析

数据准备

数据集来源：grouplens.org/datasets/movielens/
下载 ml-1m.zip，read me 中有电影评分介绍
在这里插入图片描述
MovieLens 1M电影分级。稳定的基准数据集。 6000个用户观看4000部电影时获得100万个评分。发布2/2003。

数据读取

环境：ipython notebook

读取

import pandas as pd
import numpy as np
import matplotlib.pyplot as pltuser_names=['user_id','gender','age','occupation','zip']
users=pd.read_table('users.dat',sep='::',header=None,names=user_names,engine='python')
rating_names=['user_id','movie_id','rating','timestamp']
ratings=pd.read_table('ratings.dat',sep='::',header=None,names=rating_names,engine='python')
movie_names=['movie_id','title','genres']
movies=pd.read_table('movies.dat',sep='::',header=None,names=movie_names,engine='python')users.head(5)
users.head(5)
Out[32]:
use_id	gender	age	occupation	zip
0	1	F	1	10	48067
1	2	M	56	16	70072
2	3	M	25	15	55117
3	4	M	45	7	02460
4	5	M	25	20	55455
In [33]:ratings.head(5)
Out[33]:
user_id	movie_id	rating	timestamp
0	1	1193	5	978300760
1	1	661	3	978302109
2	1	914	3	978301968
3	1	3408	4	978300275
4	1	2355	5	978824291
In [38]:
movies.head(5)
Out[38]:
movie_id	title	genres
0	1	Toy Story (1995)	Animation|Children's|Comedy
1	2	Jumanji (1995)	Adventure|Children's|Fantasy
2	3	Grumpier Old Men (1995)	Comedy|Romance
3	4	Waiting to Exhale (1995)	Comedy|Drama
4	5	Father of the Bride Part II (1995)	Comedy

合并

data = pd.merge(pd.merge(ratings, users), movies)Out[45]:
user_id	movie_id	rating	timestamp	gender	age	occupation	zip	title	genres
0	1	1193	5	978300760	F	1	10	48067	One Flew Over the Cuckoo's Nest (1975)	Drama
1	2	1193	5	978298413	M	56	16	70072	One Flew Over the Cuckoo's Nest (1975)	Drama
2	12	1193	4	978220179	M	25	12	32793	One Flew Over the Cuckoo's Nest (1975)	Drama
3	15	1193	4	978199279	M	25	7	22903	One Flew Over the Cuckoo's Nest (1975)	Drama
4	17	1193	5	978158471	M	50	1	95350	One Flew Over the Cuckoo's Nest (1975)	Dramadata[data.user_id==1]
Out[46]:
user_id	movie_id	rating	timestamp	gender	age	occupation	zip	title	genres
0	1	1193	5	978300760	F	1	10	48067	One Flew Over the Cuckoo's Nest (1975)	Drama
1725	1	661	3	978302109	F	1	10	48067	James and the Giant Peach (1996)	Animation|Children's|Musical
2250	1	914	3	978301968	F	1	10	48067	My Fair Lady (1964)	Musical|Romance
2886	1	3408	4	978300275	F	1	10	48067	Erin Brockovich (2000)	Drama
4201	1	2355	5	978824291	F	1	10	48067	Bug's Life, A (1998)	Animation|Children's|Comedy
5904	1	1197	3	978302268	F	1	10	48067	Princess Bride, The (1987)	Action|Adventure|Comedy|Romance
8222	1	1287	5	978302039	F	1	10	48067	Ben-Hur (1959)	Action|Adventure|Drama
8926	1	2804	5	978300719	F	1	10	48067	Christmas Story, A (1983)	Comedy|Drama
10278	1	594	4	978302268	F	1	10	48067	Snow White and the Seven Dwarfs (1937)	Animation|Children's|Musical
11041	1	919	4	978301368	F	1	10	48067	Wizard of Oz, The (1939)	Adventure|Children's|Drama|Musical
12759	1	595	5	978824268	F	1	10	48067	Beauty and the Beast (1991)	Animation|Children's|Musical
13819	1	938	4	978301752	F	1	10	48067	Gigi (1958)	Musical
14006	1	2398	4	978302281	F	1	10	48067	Miracle on 34th Street (1947)	Drama
14386	1	2918	4	978302124	F	1	10	48067	Ferris Bueller's Day Off (1986)	Comedy
15859	1	1035	5	978301753	F	1	10	48067	Sound of Music, The (1965)	Musical
16741	1	2791	4	978302188	F	1	10	48067	Airplane! (1980)	Comedy
18472	1	2687	3	978824268	F	1	10	48067	Tarzan (1999)	Animation|Children's
18914	1	2018	4	978301777	F	1	10	48067	Bambi (1942)	Animation|Children's
19503	1	3105	5	978301713	F	1	10	48067	Awakenings (1990)	Drama
20183	1	2797	4	978302039	F	1	10	48067	Big (1988)	Comedy|Fantasy
21674	1	2321	3	978302205	F	1	10	48067	Pleasantville (1998)	Comedy
22832	1	720	3	978300760	F	1	10	48067	Wallace & Gromit: The Best of Aardman Animatio...	Animation
23270	1	1270	5	978300055	F	1	10	48067	Back to the Future (1985)	Comedy|Sci-Fi
25853	1	527	5	978824195	F	1	10	48067	Schindler's List (1993)	Drama|War
28157	1	2340	3	978300103	F	1	10	48067	Meet Joe Black (1998)	Romance
28501	1	48	5	978824351	F	1	10	48067	Pocahontas (1995)	Animation|Children's|Musical|Romance
28883	1	1097	4	978301953	F	1	10	48067	E.T. the Extra-Terrestrial (1982)	Children's|Drama|Fantasy|Sci-Fi
31152	1	1721	4	978300055	F	1	10	48067	Titanic (1997)	Drama|Romance
32698	1	1545	4	978824139	F	1	10	48067	Ponette (1996)	Drama
32771	1	745	3	978824268	F	1	10	48067	Close Shave, A (1995)	Animation|Comedy|Thriller
33428	1	2294	4	978824291	F	1	10	48067	Antz (1998)	Animation|Children's
34073	1	3186	4	978300019	F	1	10	48067	Girl, Interrupted (1999)	Drama
34504	1	1566	4	978824330	F	1	10	48067	Hercules (1997)	Adventure|Animation|Children's|Comedy|Musical
34973	1	588	4	978824268	F	1	10	48067	Aladdin (1992)	Animation|Children's|Comedy|Musical
36324	1	1907	4	978824330	F	1	10	48067	Mulan (1998)	Animation|Children's
36814	1	783	4	978824291	F	1	10	48067	Hunchback of Notre Dame, The (1996)	Animation|Children's|Musical
37204	1	1836	5	978300172	F	1	10	48067	Last Days of Disco, The (1998)	Drama
37339	1	1022	5	978300055	F	1	10	48067	Cinderella (1950)	Animation|Children's|Musical
37916	1	2762	4	978302091	F	1	10	48067	Sixth Sense, The (1999)	Thriller
40375	1	150	5	978301777	F	1	10	48067	Apollo 13 (1995)	Drama
41626	1	1	5	978824268	F	1	10	48067	Toy Story (1995)	Animation|Children's|Comedy
43703	1	1961	5	978301590	F	1	10	48067	Rain Man (1988)	Drama
45033	1	1962	4	978301753	F	1	10	48067	Driving Miss Daisy (1989)	Drama
45685	1	2692	4	978301570	F	1	10	48067	Run Lola Run (Lola rennt) (1998)	Action|Crime|Romance
46757	1	260	4	978300760	F	1	10	48067	Star Wars: Episode IV - A New Hope (1977)	Action|Adventure|Fantasy|Sci-Fi
49748	1	1028	5	978301777	F	1	10	48067	Mary Poppins (1964)	Children's|Comedy|Musical
50759	1	1029	5	978302205	F	1	10	48067	Dumbo (1941)	Animation|Children's|Musical
51327	1	1207	4	978300719	F	1	10	48067	To Kill a Mockingbird (1962)	Drama
52255	1	2028	5	978301619	F	1	10	48067	Saving Private Ryan (1998)	Action|Drama|War
54908	1	531	4	978302149	F	1	10	48067	Secret Garden, The (1993)	Children's|Drama
55246	1	3114	4	978302174	F	1	10	48067	Toy Story 2 (1999)	Animation|Children's|Comedy
56831	1	608	4	978301398	F	1	10	48067	Fargo (1996)	Crime|Drama|Thriller
59344	1	1246	4	978302091	F	1	10	48067	Dead Poets Society (1989)	Drama

女生最喜欢的电影排行榜/男生最喜欢的电影排行榜

rating_by_gender=data.pivot_table(values='rating',index='title',columns='gender',aggfunc='mean')
gender	F	M
title		
$1,000,000 Duck (1971)	3.375000	2.761905
'Night Mother (1986)	3.388889	3.352941
'Til There Was You (1997)	2.675676	2.733333
'burbs, The (1989)	2.793478	2.962085
...And Justice for All (1979)	3.828571	3.689024
1-900 (1994)	2.000000	3.000000
10 Things I Hate About You (1999)	3.646552	3.311966
101 Dalmatians (1961)	3.791444	3.500000
101 Dalmatians (1996)	3.240000	2.911215
12 Angry Men (1957)	4.184397	4.328421

评分差

rating_by_gender['diff']=rating_by_gender.F-rating_by_gender.M
In [50]:rating_by_gender.head(10)
Out[50]:
gender	F	M	diff
title			
$1,000,000 Duck (1971)	3.375000	2.761905	0.613095
'Night Mother (1986)	3.388889	3.352941	0.035948
'Til There Was You (1997)	2.675676	2.733333	-0.057658
'burbs, The (1989)	2.793478	2.962085	-0.168607
...And Justice for All (1979)	3.828571	3.689024	0.139547
1-900 (1994)	2.000000	3.000000	-1.000000
10 Things I Hate About You (1999)	3.646552	3.311966	0.334586
101 Dalmatians (1961)	3.791444	3.500000	0.291444
101 Dalmatians (1996)	3.240000	2.911215	0.328785
12 Angry Men (1957)	4.184397	4.328421	-0.144024

评分差距最大

rating_by_gender.sort_values(by='diff',ascending=True).head(10)
Out[52]:
gender	F	M	diff
title			
Tigrero: A Film That Was Never Made (1994)	1.0	4.333333	-3.333333
Neon Bible, The (1995)	1.0	4.000000	-3.000000
Enfer, L' (1994)	1.0	3.750000	-2.750000
Stalingrad (1993)	1.0	3.593750	-2.593750
Killer: A Journal of Murder (1995)	1.0	3.428571	-2.428571
Dangerous Ground (1997)	1.0	3.333333	-2.333333
In God's Hands (1998)	1.0	3.333333	-2.333333
Rosie (1998)	1.0	3.333333	-2.333333
Flying Saucer, The (1950)	1.0	3.300000	-2.300000
Jamaica Inn (1939)	1.0	3.142857	-2.142857

评分次数

ratings_by_title=data.groupby('title').size()
In [55]:ratings_by_title.sort_values(ascending=False).head(10)
Out[55]:
title
American Beauty (1999)                                   3428
Star Wars: Episode IV - A New Hope (1977)                2991
Star Wars: Episode V - The Empire Strikes Back (1980)    2990
Star Wars: Episode VI - Return of the Jedi (1983)        2883
Jurassic Park (1993)                                     2672
Saving Private Ryan (1998)                               2653
Terminator 2: Judgment Day (1991)                        2649
Matrix, The (1999)                                       2590
Back to the Future (1985)                                2583
Silence of the Lambs, The (1991)                         2578
dtype: int64

统计电影平均得分

mean_ratings=data.pivot_table(values='rating',index='title',aggfunc='mean')
In [57]:
mean_ratings.head(10)
Out[57]:
rating
title	
$1,000,000 Duck (1971)	3.027027
'Night Mother (1986)	3.371429
'Til There Was You (1997)	2.692308
'burbs, The (1989)	2.910891
...And Justice for All (1979)	3.713568
1-900 (1994)	2.500000
10 Things I Hate About You (1999)	3.422857
101 Dalmatians (1961)	3.596460
101 Dalmatians (1996)	3.046703
12 Angry Men (1957)	4.295455
In [62]:
mean_ratings.sort_values(by='rating',ascending=False).head(20)
Out[62]:
rating
title	
Ulysses (Ulisse) (1954)	5.000000
Lured (1947)	5.000000
Follow the Bitch (1998)	5.000000
Bittersweet Motel (2000)	5.000000
Song of Freedom (1936)	5.000000
One Little Indian (1973)	5.000000
Smashing Time (1967)	5.000000
Schlafes Bruder (Brother of Sleep) (1995)	5.000000
Gate of Heavenly Peace, The (1995)	5.000000
Baby, The (1973)	5.000000
I Am Cuba (Soy Cuba/Ya Kuba) (1964)	4.800000
Lamerica (1994)	4.750000
Apple, The (Sib) (1998)	4.666667
Sanjuro (1962)	4.608696
Seven Samurai (The Magnificent Seven) (Shichinin no samurai) (1954)	4.560510
Shawshank Redemption, The (1994)	4.554558
Godfather, The (1972)	4.524966
Close Shave, A (1995)	4.520548
Usual Suspects, The (1995)	4.517106
Schindler's List (1993)	4.510417