Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102import ConfigParserimport osimport shutil
from scrapy.utils.project import get_project_settings
class Configurator: """ A helper class in the fourmi class. This class is used to process the settings as set from one of the Fourmi applications. """
def __init__(self): self.scrapy_settings = get_project_settings()
def set_output(self, filename, fileformat, compound): """ This function manipulates the Scrapy output file settings that normally would be set in the settings file. In the Fourmi project these are command line arguments. :param filename: The filename of the file where the output will be put. :param fileformat: The format in which the output will be. """
if filename != '<compound>.*format*': self.scrapy_settings.overrides["FEED_URI"] = filename elif fileformat == "jsonlines": self.scrapy_settings.overrides["FEED_URI"] = compound + ".json" elif fileformat is not None: self.scrapy_settings.overrides["FEED_URI"] = compound + "." + fileformat
if fileformat is not None: self.scrapy_settings.overrides["FEED_FORMAT"] = fileformat
def set_logging(self, logfile=None, verbose=0): """ This function changes the default settings of Scapy's logging functionality using the settings given by the CLI. :param logfile: The location where the logfile will be saved. :param verbose: A integer value to switch between loglevels. """ if verbose != 0: self.scrapy_settings.overrides["LOG_ENABLED"] = True else: self.scrapy_settings.overrides["LOG_ENABLED"] = False
if verbose == 1: self.scrapy_settings.overrides["LOG_LEVEL"] = "WARNING" elif verbose == 2: self.scrapy_settings.overrides["LOG_LEVEL"] = "INFO" else: self.scrapy_settings.overrides["LOG_LEVEL"] = "DEBUG"
if verbose > 1: self.scrapy_settings.overrides["LOG_STDOUT"] = False else: self.scrapy_settings.overrides["LOG_STDOUT"] = True
if logfile is not None: self.scrapy_settings.overrides["LOG_FILE"] = logfile else: self.scrapy_settings.overrides["LOG_FILE"] = None
@staticmethod def read_sourceconfiguration(): """ This function reads sources.cfg in the main folder for configuration variables for sources :return a ConfigParser object of sources.cfg """ current_dir = os.path.dirname(os.path.abspath(__file__)) config_path = current_dir + '/../sources.cfg' # [TODO]: location of sources.cfg should be softcoded eventually if not os.path.isfile(config_path): try: shutil.copyfile(os.path.dirname(os.path.abspath(__file__)) + "/../sources.cfg.sample", config_path) except IOError: print "WARNING: Source configuration couldn't be found and couldn't be created." config = ConfigParser.ConfigParser() config.read(config_path) return config
@staticmethod def get_section(config, sourcename): """ This function reads a config section labeled in variable sourcename and tests whether the reliability variable is set else set to empty string. Return the default section if the labeled config section does not exist :param config: a ConfigParser object :param sourcename: the name of the section to be read :return a dictionary of the section in the config labeled in sourcename """ section = dict() if config.has_section(sourcename): section = dict(config.items(sourcename)) elif config.defaults(): section = config.defaults() if 'reliability' not in section: print 'WARNING: Reliability not set for %s' % sourcename section['reliability'] = '' return section