# -*- coding: latin-1 -*- # # Copyright 2008-2009 Jesper ÷qvist (http://llbit.se/) # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # # # Modified 11/5/09 Kyle Lad: grabs pages like a crawler. most regexps working, # certainly not all. basic page stats available, no link stats yet. 1:30pm-5:30pm. # # Modified 11/8/09 Kyle Lad: forked fetch and created fetchSeed, abstracted the # creation of the list of math pages and data files. Might look at BeautifulSoup # to parse each page by section for Link_Info, regex approach seems naive and # limited. 11am-1pm. # # Modified 11/10/09 Kyle Lad: rewrote fetch to use split(), works great! No need # for soup. Now count page stats by summing section stats, less redundancy. # Running code overnight. 7:00-8:30pm. # # Modified 11/11/09 Kyle Lad: Slight changes to skip_regex, basic link scoring # (though I'm not happy with the result). Wrote README. 6:30-7:30pm. # # Modified 4/4/13 Greg Leibon: Made it Mac and new Wiki compaitible import httplib import socket import sys import os.path import re class Page: def __init__(self, name): self.name = name self.refs = [] self.sections = [] def __str__(self): return self.name def read(self): # get page content, denotated by HTML comments global http_headers, domain conn = httplib.HTTPConnection(domain) conn.request("GET", "/wiki/"+self.name, headers=http_headers) data = conn.getresponse().read() start = '' end = '' cstart = data.find(start)+len(start) cend = data.rfind(end) return data[cstart:cend] def filter(self, target): # kill any absolutely undesirable links, we won't even consider them. if target in self.refs: return False self.refs.append(target) for re in skip_regex: if re.match(target): return False return True def fetchSeed(self): global page_regex try: data = self.read() links = [ m.group(1) for m in page_regex.finditer(data) ] links = filter(self.filter,links) for target in links: linkList.append(target) except BaseException, e: print "error while fetching seed "+self.name+" ("+str(e)+")" def fetch(self): try: data = self.read() if "," in self.name: self.name = self.name.replace(",","") # Link_Info section Link_Info = open("data/Link_Info.csv","a") self.sections = re.split(section_regex,data) numSections = 0 for section in self.sections: numSections += 1 sectionSize = len(section) sectionFormulae = [m.group(0) for m in formulae_regex.finditer(section)] numSectionFormulae = len(sectionFormulae) sectionLinks = [m.group(1) for m in page_regex.finditer(section)] numAllSectionLinks = len(sectionLinks) # print "["+str(numSections)+"]"+str(sectionLinks) numWikimathSectionLinks = 0 for sLink in sectionLinks: if sLink in linkList: numWikimathSectionLinks += 1 # THERE HAS GOT TO BE A BETTER WAY TO COUNT THIS for sLink in sectionLinks: if sLink in linkList: sLinksW=1.0-1.0/float(numAllSectionLinks) wLinksW=1.0-1.0/float(numWikimathSectionLinks) sSizeW=1.0-100.0/float(sectionSize) #169 is smallest section size encountered so far if (numSectionFormulae == 0): nFormsW=0 else: nFormsW=1.0-1.0/float(numSectionFormulae) sWeight = 1.0/numSections SCORE = 5.0*sWeight+4.0*wLinksW+3.0*sSizeW+2.0*sLinksW+1.0*nFormsW if "," in sLink: sLink = sLink.replace(",","") Link_Info.write(self.name + "," + sLink + "," + str(numAllSectionLinks)+","+\ str(numWikimathSectionLinks) + "," +str(numSections)+","+\ str(sectionSize) + "," + str(numSectionFormulae) + "," +\ str(sWeight)+","+str(wLinksW)+","+str(sSizeW)+","+str(sLinksW)+","+\ str(nFormsW)+","+str(SCORE)+"\n") Link_Info.close() # Page_Info section Page_Info = open("data/Page_Info.csv","a") links = [ m.group(1) for m in page_regex.finditer(data) ] numUnfilteredPageLinks = len(links) links = filter(self.filter, links) numFilteredPageLinks = len(links) numPageChars = len(data) formulae = [ m.group(0) for m in formulae_regex.finditer(data) ] numFormulae = len(formulae) Page_Info.write(str(self.name) + ",") Page_Info.write(str(numUnfilteredPageLinks)+",") Page_Info.write(str(numFilteredPageLinks)+",") Page_Info.write(str(numPageChars)+",") Page_Info.write(str(numFormulae)+",") Page_Info.write(str(numSections)+"\n") Page_Info.close() except BaseException, e: print "error while fetching "+self.name+" ("+str(e)+")" def getPage(pageName): print "fetching page:", pageName page = Page(pageName) page.fetch() print "done" def getSeedPage(pageName): print "Adding seed page:", pageName page = Page(pageName) page.fetchSeed() print "done" def createFiles(PI_cats, LI_cats): if not os.path.exists("data"): os.mkdir("data") if os.path.exists("data/Page_Info.csv"): os.remove("data/Page_Info.csv") if os.path.exists("data/Link_Info.csv"): os.remove("data/Link_Info.csv") Page_Info = open("data/Page_Info.csv", "w") Page_Info.write(PI_cats) Page_Info.close() Link_Info = open("data/Link_Info.csv", "w") Link_Info.write(LI_cats) Link_Info.close() if __name__ == "__main__": linkList = [] # list of permissible pages to explore user_agent = "monkey" http_headers = {"User-Agent": user_agent} lang_prefix = "en" domain = lang_prefix+".wikipedia.org" timeout = 60 socket.setdefaulttimeout(timeout) # Compile regexps # Looking for links to other wiki pages. # First group () contains the link without the leading /wiki/ part # Second group () is OPTIONAL and if it exists, it stores any RELATIVE link (to another section on the same page) # I think we are doing this so that the first group ALWAYS contains what we want to get at. # Including the second grouping is just a hack to force our URLs to remove any section-specific linking page_regex = re.compile(r'