okfn-brasil/serenata-de-amor

import json
import requests
import re
import os.path
import datetime
from unicodedata import normalize

from decouple import config
import numpy as np
import pandas as pd
from pandas.io.json import json_normalize


"""
Get your API access token

1. Create an Yelp account.
2. Create an app (https://www.yelp.com/developers/v3/manage_app).
3. Run this command in your terminal to get yout access_token:
  curl -X POST -F 'client_id=YOUR_CLIENT_ID' -F 'client_secret=YOUT_CLIENT_SECRET' https://api.yelp.com/oauth2/token
4. Get your 'access_token' from the response and add it as an environment variable.
"""


def companies():
    # Loading reimbursements
    docs = pd.read_csv(REIMBURSEMENTS_DATASET_PATH,
                       low_memory=False,
                       dtype={'cnpj_cpf': np.str})
    # Filtering only congressperson meals
    meal_docs = docs[docs.subquota_description == 'Congressperson meal']
    # Storing only unique CNPJs
    meal_cnpjs = meal_docs['cnpj_cpf'].unique()
    # Loading companies
    all_companies = pd.read_csv(COMPANIES_DATASET_PATH,
                                low_memory=False,
                                dtype={'trade_name': np.str})
    all_companies = all_companies[all_companies['trade_name'].notnull()]
    # Cleaning up companies CNPJs
    all_companies['clean_cnpj'] = \
        all_companies['cnpj'].map(lambda cnpj: cnpj.replace(
            '.', '').replace('/', '').replace('-', ''))
    # Filtering only companies that are in meal reimbursements
    return all_companies[all_companies['clean_cnpj'].isin(meal_cnpjs)]


def find_newest_file(name):
    date_regex = re.compile('\d{4}-\d{2}-\d{2}')

    matches = (date_regex.findall(f) for f in os.listdir(DATA_DIR))
    dates = sorted(set([l[0] for l in matches if l]), reverse=True)

    for date in dates:
        filename = '{}-{}.xz'.format(date, name)
        filepath = os.path.join(DATA_DIR, filename)

        if os.path.isfile(filepath):
            return filepath


def remaining_companies(fetched_companies, companies):
    return companies[~companies['cnpj'].isin(fetched_companies['cnpj'])]


def load_companies_dataset():
    if os.path.exists(YELP_DATASET_PATH):
        return pd.read_csv(YELP_DATASET_PATH)
    else:
        return pd.DataFrame(columns=['cnpj'])


def parse_fetch_info(response):
    if response.status_code == 200:
        json = response.json()
        results = json['businesses']
        if results:
            return results[0]
    else:
        print('Response ==>', response.status_code)


def write_fetched_companies(companies):
    companies.to_csv(YELP_DATASET_PATH,
                     compression='xz',
                     index=False)
# ----------------------------
# Request to yelp API getting by trade name and address
# https://www.yelp.com/developers/documentation/v3/business_search


def fetch_yelp_info(**params):
    url = 'https://api.yelp.com/v3/businesses/search'
    authorization = "Bearer {}".format(config('YELP_ACCESS_TOKEN'))
    headers = {"Authorization": authorization}
    response = requests.get(url, headers=headers, params=params)
    return parse_fetch_info(response)


def standardize_name(name):
    new_name = normalize('NFKD', name).encode(
        'ASCII', 'ignore').decode('utf-8')
    return set(new_name.lower().split(' '))


DATA_DIR = 'data'
REIMBURSEMENTS_DATASET_PATH = find_newest_file('reimbursements')
COMPANIES_DATASET_PATH = find_newest_file('companies')
YELP_DATASET_PATH = os.path.join('data', 'yelp-companies.xz')


if __name__ == '__main__':
    companies_w_meal_expense = companies()
    fetched_companies = load_companies_dataset()
    companies_to_fetch = remaining_companies(
        fetched_companies, companies_w_meal_expense).reset_index()

    for index, company in companies_to_fetch.iterrows():
        print('%s: Fetching %s - City: %s' %
              (index, company['trade_name'], company['city']))

        fetched_company = fetch_yelp_info(term=company['trade_name'],
                                          location='BR',
                                          latitude=company['latitude'],
                                          longitude=company['longitude'])

        is_good_result = False
        if fetched_company:
            expected_name = standardize_name(company['trade_name'])
            result_name = standardize_name(fetched_company['name'])
            is_good_result = len(
                expected_name - result_name) / len(expected_name) < .3

        if is_good_result:
            print('Successfuly matched %s' % fetched_company['name'])
        else:
            print('Not found')
            fetched_company = {}

        record = json_normalize(fetched_company)
        record['scraped_at'] = datetime.datetime.utcnow().isoformat()
        record['trade_name'] = company['trade_name']
        record['cnpj'] = company['cnpj']
        fetched_companies = pd.concat([fetched_companies, record])

        if (index % 100) == 0 and index > 0:
            print('###########################################')
            print("%s requests made. Stopping to save." % index)
            write_fetched_companies(fetched_companies)
            print('###########################################')

    write_fetched_companies(fetched_companies)