MaxApp/populate.py

from datetime import datetime
from tqdm import tqdm 
from pandas import DataFrame
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from pycoingecko import CoinGeckoAPI
from Coin import Coin, CoinMarketPosition
from requests import HTTPError
import time
from multiprocessing import Pool

DATABASE_URI = 'sqlite:///coins.db'
engine = create_engine('sqlite:///coins.db', echo=False)

# create a Session
Session = sessionmaker(bind=engine)
session = Session()

cg = CoinGeckoAPI()
def importMarketPosition(coin:Coin):
    engine = create_engine(DATABASE_URI, echo=False)

    # create a Session
    Session = sessionmaker(bind=engine)
    session = Session()

    # skip already bootstraped items
    if session.query(CoinMarketPosition).filter(CoinMarketPosition.coin_id == coin.row_id).first() is not None:
        return
    try:
        positions = cg.get_coin_market_chart_by_id(coin.id,"usd","max")
    except (HTTPError) as e:
        print("We wait 60s :(", e.response)
        time.sleep(61)
        positions = cg.get_coin_market_chart_by_id(coin.id,"usd","max")

    df_prices = DataFrame(positions["prices"])
    df_prices.columns=["date","prices"]

    df_total_volumes = DataFrame(positions["total_volumes"])
    df_total_volumes.columns=["date","total_volumes"]

    df_market_caps = DataFrame(positions["market_caps"])
    df_market_caps.columns=["date","market_caps"]

    df = df_prices.set_index('date').join(df_market_caps.set_index('date'),"date",'outer').join(df_total_volumes.set_index('date'),"date",'outer')

    for t,row in df.iterrows():
        try:
            market_position = CoinMarketPosition(coin.row_id,datetime.fromtimestamp(t/1000),row.total_volumes,row.market_caps,row.prices)
            session.add(market_position)
        except OSError as e:
            print("ERROR", "Invalid data for coin ", coin.id,row.total_volumes,row.market_caps,row.prices)

    session.commit()


if __name__ == "__main__":
    # Create objects
    for c in cg.get_coins_list():
        obj = Coin (c["id"], c["symbol"], c["name"])
        if session.query(Coin).filter(Coin.id == obj.id).first() is None:
            session.add(obj)
    # Save the change made on the database.            
    session.commit()

    # collect market position  
    with Pool(10) as p:
        coin_list = session.query(Coin).order_by(Coin.row_id).all()
        r = list(tqdm(p.imap(importMarketPosition, coin_list), total=len(coin_list)))