西电教务系统
- Grades Crawler 项目介绍
由于教务处成绩不易自己打印，故编写爬虫Grades Crawler, 其可以将自己的（需要合法的学生帐号信息）教务处成绩爬取下来，并本地保存为excel格式或直接保存在sqlite数据库中
需额外安装的 libraries
bs4, xlrd, xlwt(保存成绩为excel格式)
# -*-encoding:utf-8-*-
# coding=utf-8
__author__ = 'ysc'
import requests
from bs4 import BeautifulSoup
import xlrd
import xlwt
class ScrapeGrade:
def __init__(self, auth_url=None, log_url=None):
if not auth_url:
self.auth_url = "http://ids.xidian.edu.cn/authserver/login?service=http%3A%2F%2Fjwxt.xidian.edu.cn%2Fcaslogin.jsp"
self.log_url = "http://jwxt.xidian.edu.cn/caslogin.jsp"
else:
self.auth_url = auth_url
self.log_url = log_url
self.session = requests.Session()
def login(self, id='1302051****', password='****'):
r = self.session.get(self.auth_url)
data = r.text
bsObj = BeautifulSoup(data, "html.parser")
lt_value = bsObj.find(attrs={"name": "lt"})['value']
exe_value = bsObj.find(attrs={"name": "execution"})['value']
params = {'username': id, 'password': password,
"submit": "", "lt": lt_value, "execution": exe_value,
"_eventId": "submit", "rmShown": '1'}
headers = {'User-Agent': "Mozilla/5.0 (windows NT 10.0; WOW64; rv:44.0) Gecko/20100101 Firefox/44.0",
'Accept': "text/html,APPlication/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
# "Host": "ids.xidian.edu.cn",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
"Accept-Encoding": "gzip, deflate",
"referer": "http://ids.xidian.edu.cn/authserver/login?service=http%3A%2F%2Fjwxt.xidian.edu.cn%2Fcaslogin.jsp",
# 'X-Requested-With': "XMLHttpRequest",
"content-Type": "application/x-www-form-urlencoded"}
s = self.session.post(self.auth_url, data=params, headers=headers)
s = self.session.get(self.log_url)
def store_into_db_by_term(self):
# 按学期进行分类
import sqlite3
conn = sqlite3.connect('grades_term.db')
# conn.text_factory = str ##!!!
c = conn.cursor()
try:
# init the counter of the sheet
row = 0
# 打开成绩页面
grade_page = self.session.get("http://jwxt.xidian.edu.cn/gradeLnAllAction.do?type=ln&oper=qbinfo&lnxndm=2015-2016%D1%A7%C4%EA%B5%DA%D2%BB%D1%A7%C6%DA(%C1%BD%D1%A7%C6%DA)")
bsObj2 = BeautifulSoup(grade_page.text, "html.parser")
# datas 包含了所有学期的成绩, table
datas = bsObj2.find_all("table", attrs={"class": "titleTop2"})
# seme 指每学期的成绩. table
for i, seme in enumerate(datas):
#写入一行标题th
ths = seme.find_all('th')
titles = []
for col, th in enumerate(ths):
print(th.string.strip(), end=' ')
th = th.string.strip()
if th != '学分' and th != "成绩":
titles.append(th + r' text')
else:
titles.append(th + r' real')
# table.write(row, col, th.string.strip(), self.set_style('Times New Roman', 220, True))
# Create table
sent = '''CREATE TABLE {0} ( '''.format('table' + str(i+1))
for ith, title in enumerate(titles):
sent += title
if ith < len(titles) - 1:
sent += ", "
sent += ")"
try:
c.execute(sent)
conn.commit()
except sqlite3.OperationalERROR:
pass
print('\n')
row += 1
# 各科成绩
subs = seme.findAll('td', attrs={"align": "center"})
col_iter = 0
len_ths = len(ths)
grade_subs = []
# sub为具体的某科成绩
for sub in subs:
if sub.string:
if sub.string.strip() != '':
print(sub.string.strip(), end=' ')
grade_subs.append("'" + sub.string.strip()+"'")
else:
print("' '", end=' ')
grade_subs.append("' '")
else:
print(sub.find('p').string.strip(), end=' ')
grade_subs.append("'" + sub.find('p').string.strip() + "'")
col_iter += 1
if col_iter == len_ths:
# 此时一科的成绩以及visited, 该访问下一科
print('\n')
# Insert a row of data
sent = '''INSERT INTO {0} VALUES( '''.format('table' + str(i+1))
for ith, grade_sub in enumerate(grade_subs):
sent += grade_sub
if ith < len(grade_subs) - 1:
sent += ", "
sent += ")"
try:
c.execute(sent)
conn.commit()
except sqlite3.OperationalError as e:
print(e)
print(sent)
exit(-2)
row += 1
col_iter = 0
grade_subs = []
print("\n")
# 保存到xls中
finally:
conn.close()
def store_into_db_by_prop(self):
# 按科目属性(必修\选修)进行分类
import sqlite3
conn = sqlite3.connect('grades_prop.db')
c = conn.cursor()
try:
# init the counter of the sheet
row = 0
# 打开成绩页面
grade_page = self.session.get("http://jwxt.xidian.edu.cn/gradeLnAllAction.do?type=ln&oper=sxinfo&lnsxdm=001")
bsObj2 = BeautifulSoup(grade_page.text, "html.parser")
# datas 包含了所有学期的成绩, table
datas = bsObj2.find_all("table", attrs={"class": "titleTop2"})
# seme 指每学期的成绩. table
for i, seme in enumerate(datas):
#写入一行标题th
ths = seme.find_all('th')
titles = []
for col, th in enumerate(ths):
print(th.string.strip(), end=' ')
th = th.string.strip()
if th != '学分' and th != "成绩":
titles.append(th + r' text')
else:
titles.append(th + r' real')
# table.write(row, col, th.string.strip(), self.set_style('Times New Roman', 220, True))
# Create table
sent = '''CREATE TABLE {0} ( '''.format('table' + str(i+1))
for ith, title in enumerate(titles):
sent += title
if ith < len(titles) - 1:
sent += ", "
sent += ")"
try:
c.execute(sent)
conn.commit()
except sqlite3.OperationalError:
pass
print('\n')
row += 1
# 各科成绩
subs = seme.findAll('tr', attrs={'class': "odd"})
col_iter = 0
len_ths = len(ths)
grade_subs = []
# sub为具体的某科信息
for sub in subs:
infors = sub.findAll('td') #, attrs={"align": "center"})
for infor in infors:
if infor.string:
if infor.string.strip() != '':
print(infor.string.strip(), end=' ')
grade_subs.append("'" + infor.string.strip()+"'")
else:
print("' '", end=' ')
grade_subs.append("' '")
else:
infor = infor.find('p').string.strip()
if infor != '':
print(infor, end=' ')
grade_subs.append("'" + infor + "'")
else:
print("' '", end=' ')
grade_subs.append("' '")
# 此时一科的成绩已经visited, 该访问下一科
print('\n')
# Insert a row of data
sent = '''INSERT INTO {0} VALUES( '''.format('table' + str(i+1))
for ith, grade_sub in enumerate(grade_subs):
sent += grade_sub
if ith < len(grade_subs) - 1:
sent += ", "
sent += ")"
try:
c.execute(sent)
conn.commit()
except sqlite3.OperationalError as e:
print(e)
print(sent)
exit(-2)
row += 1
col_iter = 0
grade_subs = []
print("\n")
# 保存到xls中
finally:
conn.close()
def set_style(self, name, height, bold=False):
style = xlwt.XFStyle()
font = xlwt.Font()
font.name = name # 'Times New Roman'
font.bold = bold
font.color_index = 4
font.height = height
'''
borders= xlwt.Borders()
borders.left= 6
borders.right= 6
borders.top= 6
borders.bottom= 6
'''
style.font = font
# style.borders = borders
return style
def store_into_xls(self):
file = xlwt.Workbook()
table = file.add_sheet('grades', cell_overwrite_ok=True)
# init the counter of the sheet
row = 0
# 打开成绩页面
grade_page = self.session.get("http://jwxt.xidian.edu.cn/gradeLnAllAction.do?type=ln&oper=qbinfo&lnxndm=2015-2016%D1%A7%C4%EA%B5%DA%D2%BB%D1%A7%C6%DA(%C1%BD%D1%A7%C6%DA)")
bsObj2 = BeautifulSoup(grade_page.text, "html.parser")
# datas 包含了所有学期的成绩, table
datas = bsObj2.find_all("table", attrs={"class": "titleTop2"})
# seme 指每学期的成绩. table
for seme in datas:
#写入一行标题th
ths = seme.find_all('th')
for col, th in enumerate(ths):
print(th.string.strip(), end=' ')
table.write(row, col, th.string.strip(), self.set_style('Times New Roman', 220, True))
print('\n')
row += 1
# 各科成绩
subs = seme.findAll('td', attrs={"align": "center"})
col_iter = 0
len_ths = len(ths)
# sub为具体的某科成绩
for sub in subs:
if sub.string:
print(sub.string.strip(), end=' ')
table.write(row, col_iter, sub.string.strip())
else:
print(sub.find('p').string.strip(), end=' ')
table.write(row, col_iter, sub.find('p').string.strip())
col_iter += 1
if col_iter == len_ths:
print('\n')
row += 1
col_iter = 0
print("\n")
# 保存到xls中
file.save('demo.xls')
if __name__ == '__main__':
# 初始化爬虫对象
sg = ScrapeGrade()
# 登录(在此处传入正确的个人学号与密码信息)
sg.login(id='1302051***', password='1234567')
# 保存成绩为excel
sg.store_into_xls()
文章最后发布于: 2016-04-25 22:38:08
相关阅读
A5创业网（公众号：iadmin5）1月15日报道，近日北京监狱与支付宝合作，上线了服刑人员综合账务管理系统，为服刑人员提供了狱内支付、家属存款
营销QQ是拥有10万好友容量的一个企业版QQ，可以每天发出1000次好友邀请，但是很多用户在购买使用之后，发现并不能达到1000次邀请量，经常
浅谈setInterval(aa,1000)与setInterval(aa(),1000)的
一直有个疑惑，在定时器上调用某个方法时，加括号和不加括号有什么区别。今天做了个实验，发现，不加括号定时器会每秒执行一次，加了括号只
图片来源图虫：已授站长之家使用声明：本文来自于微信公众号运营研究社公众号(ID：U_quan)，作者：陈维贤，授权站长之家转载发布。文章整理自
与E1000E和E1000相比，VMXNET3的网络性能更好。本文将解释虚拟网络适配器和第2部分之间的区别，并将演示通过选择半虚拟化适配器可以