Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
6.1 kB · 149 lines
Python
at develop
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150import re
from scrapy.http import Requestfrom scrapy import logfrom scrapy.selector import Selector
from source import Sourcefrom FourmiCrawler.items import Result
class PubChem(Source): """ PubChem scraper for chemical properties
This parser parses the part on PubChem pages that gives Chemical and Physical properties of a substance, including sources of the values of properties. """
# PubChem has its data on compound name, properties and their values on different html pages, so different URLs used website = 'http://.*\\.ncbi\\.nlm\\.nih\\.gov/.*' website_www = 'http://www.ncbi.nlm.nih.gov/*' website_pubchem = 'http://pubchem.ncbi.nlm.nih.gov/.*' search = 'pccompound?term=%s' data_url = 'toc/summary_toc.cgi?tocid=27&cid=%s'
__spider = None searched_compounds = set()
def __init__(self, config): Source.__init__(self, config) self.cfg = config
def parse(self, response): """ Distributes the above described behaviour :param response: The incoming search request :return Returns the found properties if response is unique or returns none if it's already known """ requests = [] log.msg('A response from %s just arrived!' % response.url, level=log.DEBUG)
sel = Selector(response) compound = sel.xpath('//h1/text()').extract()[0] if compound in self.searched_compounds: return None
self.searched_compounds.update(compound) raw_synonyms = sel.xpath('//div[@class="smalltext"]/text()').extract()[0] for synonym in raw_synonyms.strip().split(', '): log.msg('PubChem synonym found: %s' % synonym, level=log.DEBUG) self.searched_compounds.update(synonym) self._spider.get_synonym_requests(synonym) log.msg('Raw synonyms found: %s' % raw_synonyms, level=log.DEBUG)
n = re.search(r'cid=(\d+)', response.url) if n: cid = n.group(1) log.msg('cid: %s' % cid, level=log.DEBUG) # getting the right id of the compound with which it can reach # the seperate html page which contains the properties and their values
# using this cid to get the right url and scrape it requests.append( Request(url=self.website_pubchem[:-2].replace("\\", "") + self.data_url % cid, callback=self.parse_data)) return requests
def parse_data(self, response): """ Parse data found in 'Chemical and Physical properties' part of a substance page. :param response: The response with the page to parse :return: requests: Returns a list of properties with their values, source, etc. """ log.msg('parsing data', level=log.DEBUG) requests = []
sel = Selector(response) props = sel.xpath('//div')
for prop in props: prop_name = ''.join(prop.xpath('b/text()').extract()) # name of property that it is parsing if prop.xpath('a'): # parsing for single value in property prop_source = ''.join(prop.xpath('a/@title').extract()) prop_value = ''.join(prop.xpath('a/text()').extract()) new_prop = Result({ 'attribute': prop_name, 'value': prop_value, 'source': prop_source, 'reliability': self.cfg['reliability'], 'conditions': '' }) log.msg('PubChem prop: |%s| |%s| |%s|' % (new_prop['attribute'], new_prop['value'], new_prop['source']), level=log.DEBUG) requests.append(new_prop) elif prop.xpath('ul'): # parsing for multiple values (list) in property prop_values = prop.xpath('ul//li') for prop_li in prop_values: prop_value = ''.join(prop_li.xpath('a/text()').extract()) prop_source = ''.join(prop_li.xpath('a/@title').extract()) new_prop = Result({ 'attribute': prop_name, 'value': prop_value, 'source': prop_source, 'reliability': self.cfg['reliability'], 'conditions': '' }) log.msg('PubChem prop: |%s| |%s| |%s|' % (new_prop['attribute'], new_prop['value'], new_prop['source']), level=log.DEBUG) requests.append(new_prop)
return requests
def parse_searchrequest(self, response): """ This function parses the response to the new_compound_request Request :param response: the Response object to be parsed :return: A Request for the compound page or what self.parse returns in case the search request forwarded to the compound page """
# check if pubchem forwarded straight to compound page m = re.match(self.website_pubchem, response.url) if m: log.msg('PubChem search forwarded to compound page', level=log.DEBUG) return self.parse(response)
sel = Selector(response)
results = sel.xpath('//div[@class="rsltcont"]') if results: url = results[0].xpath('div/p/a[1]/@href') else: log.msg('PubChem search found nothing or xpath failed', level=log.DEBUG) return None
if url: url = 'http:' + ''.join(url[0].extract()) log.msg('PubChem compound page: %s' % url, level=log.DEBUG) else: log.msg('PubChem search found results, but no url in first result', level=log.DEBUG) return None
return Request(url=url, callback=self.parse)
def new_compound_request(self, compound): return Request(url=self.website_www[:-1] + self.search % compound, callback=self.parse_searchrequest)