Merge branch 'master' of http://gitlab.jaquin.fr/clovis/bdlg2023-sms-back

Readme.md

@@ -21,7 +21,7 @@ Note, be sure to use python3.11 with this application
 
 ## Debug
 
-> python -m app.debug.py
+> python -m app.debug
 
 Run tests
 

app/api/endpoints/project.py

@@ -1,4 +1,4 @@
-from datetime import timedelta
+from datetime import timedelta, datetime, timezone
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, HTTPException
@@ -7,13 +7,14 @@ from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from app.api import deps
-from app.models import Project, Slot, Sms, User
+from app.models import Project, Slot, Sms, User, Volunteer
 from app.schemas.requests import (
     ProjectImportGsheetRequest,
     ProjectRequest,
     ProjectSMSBatchRequest,
 )
 from app.schemas.responses import ProjectListResponse, ProjectResponse, SMSResponse
+from app.gsheet import parseGsheet, extract_doc_uid
 
 router = APIRouter()
 
@@ -49,7 +50,7 @@ async def create_project(
     try:
         session.commit()
     except IntegrityError as e:
-        raise HTTPException(422, "Project name already exist")
+        raise HTTPException(400, "Project name already exist")
 
     session.refresh(project)
 
@@ -109,8 +110,60 @@ async def update_project_from_gsheet(
     p = session.get(Project, project_id)
     if p is None:
         raise HTTPException(status_code=404, detail="Project not found")
-    url = gsheet.sheet_url
-    # TODO implement feature to import
+    doc_id = extract_doc_uid(gsheet.sheet_url)
+    if gsheet.erase_data:
+        p.slots = []
+        p.sms = []
+        p.volunteers = []
+    df_contact, df_creneau, df_planning = parseGsheet(doc_id, gsheet.satursday_date)
+    # Create the volunteer list
+    volunteer_map: dict[str, Volunteer] = {}
+    for _, row in df_contact.iterrows():
+        volunteer = Volunteer(
+            project_id=project_id,
+            name=row["Prénom"],
+            surname=row["Nom"],
+            email=row["Mail"],
+            phone_number=row["Tél"],
+            automatic_sms=row["SMS"] == "Oui",
+        )
+        volunteer_map[row.key] = volunteer
+        session.add(volunteer)
+
+    creneau_names = df_creneau.nom.unique()
+    # group planning entry per same name and timing
+    date_format = "%Y/%m/%d %H:%M"
+    df_planning["key"] = (
+        df_planning.nom.str.strip()
+        + "_"
+        + df_planning.start.dt.strftime(date_format)
+        + "-"
+        + df_planning.end.dt.strftime(date_format)
+    )
+    df_slots = df_planning.groupby("key")
+    # Create slots
+    for key in df_slots.groups.keys():
+        group = df_slots.get_group(key)
+        slot = Slot(
+            project_id=project_id,
+            title=group.nom.iloc[0],
+            starting_time=group.start.iloc[0],
+            ending_time=group.end.iloc[0],
+        )
+        # Add volunteer to slots
+        for benevole_key in group.benevole_nom.tolist():
+            if benevole_key in volunteer_map:
+                slot.volunteers.append(volunteer_map[benevole_key])
+        # add detail information if available
+        description_id = group.description_id.iloc[0]
+        if description_id in creneau_names:
+            item = df_creneau[df_creneau.nom == description_id].iloc[0]
+            slot.description = item.description
+            slot.place = item.lieu
+            slot.responsible_contact = item.responsable
+        session.add(slot)
+    session.commit()
+    session.refresh(p)
     return p
 
 
@@ -128,6 +181,7 @@ async def create_sms_batch(
      - {description} slot.description
      - {debut} slot.starting_time
      - {fin} slot.ending_ting
+     - {respo} slot.responsible_contact
      - {prenom} volunteer.name
      - {nom} volunteer.surname
 
@@ -138,6 +192,7 @@ async def create_sms_batch(
     # Get all slots
     slots = session.execute(select(Slot).where(Slot.project_id == project_id))
     sms_list = []
+    now = datetime.now(timezone.utc)
     for slot in slots.scalars():
         # Replace the slot placeholder by their value
         slot_content = (
@@ -145,15 +200,19 @@ async def create_sms_batch(
             .replace("{description}", slot.description)
             .replace("{debut}", slot.starting_time.strftime("%Hh%M"))
             .replace("{fin}", slot.ending_time.strftime("%Hh%M"))
+            .replace("{respo}", slot.responsible_contact)
         )
         sending_time = slot.starting_time - timedelta(minutes=sms_batch.delta_t)
+        # Skip SMS that should have been send before now
+        if sending_time < now:
+            continue
         for volunteer in slot.volunteers:
             if not volunteer.automatic_sms:
                 continue
             # Create a new SMS customized for each user attache to the slot
-            personalized_content = slot_content.replace(
-                "{prenom}", volunteer.name
-            ).replace("{nom}", volunteer.surname)
+            personalized_content = slot_content.replace("{prenom}", volunteer.name).replace(
+                "{nom}", volunteer.surname
+            )
             sms = Sms(
                 project_id=project_id,
                 volunteer_id=volunteer.id,

app/gsheet.py

@@ -0,0 +1,213 @@
+import datetime
+import io
+import os
+from enum import Enum
+from urllib.parse import urlparse
+from uuid import uuid4
+
+import pandas as pd
+import requests
+
+planning_gid = "1001381542"
+creneau_gid = "1884137958"
+benevole_gid = "82247394"
+
+
+class ParserState(Enum):
+    STARTING_VALUE = 0
+    READING_VALUE = 1
+    ESCAPING = 2
+    CLOSING_ESCAPE = 3
+
+
+def split_csv_row(raw_data: str, separator: str = ",", escape: str = '"') -> list[str]:
+    state: ParserState = ParserState.STARTING_VALUE
+    arr = []
+    current_item = ""
+    for c in raw_data:
+        if state == ParserState.STARTING_VALUE:
+            if c == escape:
+                state = ParserState.ESCAPING
+            elif c == separator:
+                arr.append("")
+            else:
+                state = ParserState.READING_VALUE
+                current_item = c
+        elif state == ParserState.READING_VALUE:
+            if c == separator:
+                state = ParserState.STARTING_VALUE
+                arr.append(current_item)
+                current_item = ""
+            else:
+                current_item += c
+        elif state == ParserState.ESCAPING:
+            if c == escape:
+                state = ParserState.CLOSING_ESCAPE
+            else:
+                current_item += c
+        elif state == ParserState.CLOSING_ESCAPE:
+            if c == escape:
+                state = ParserState.ESCAPING
+                current_item += c
+            else:
+                state = ParserState.READING_VALUE
+                arr.append(current_item)
+                current_item = ""
+    arr.append(current_item)
+    return arr
+
+
+class InvalidUrlError(Exception):
+    pass
+
+
+def extract_doc_uid(url: str) -> str:
+    res = urlparse(url)
+    if res.netloc != "docs.google.com":
+        raise InvalidUrlError("Invalid netloc")
+    if not res.path.startswith("/spreadsheets/d/"):
+        raise InvalidUrlError("Invalid path")
+    doc_id = res.path.split("/")[3]
+    l_doc_id = len(doc_id)
+    if l_doc_id < 32 and 50 > l_doc_id:
+        raise InvalidUrlError("Invalid path")
+    return doc_id
+
+
+def build_sheet_url(doc_id, sheet_id):
+    return f"https://docs.google.com/spreadsheets/d/{doc_id}/export?format=csv&gid={sheet_id}"
+
+
+def downloadAndSave(doc_ui, sheet_gid, fname):
+    url = build_sheet_url(doc_ui, sheet_gid)
+    print("Downloading " + fname)
+    rep = requests.get(url)
+    with open(fname, "wb") as f:
+        f.write(rep.content)
+
+
+def getContactDataFrame(csv_filename: str, skiprows: int = 2) -> pd.DataFrame:
+    df_contact = pd.read_csv(csv_filename, skiprows=skiprows)
+    column_to_drop = [name for name in df_contact.columns if "Unnamed" in name]
+    df_contact.drop(column_to_drop, axis=1, inplace=True)
+
+    # Filter out empty name
+    df_contact = df_contact[~df_contact.Nom.isnull()]
+    df_contact.reset_index()
+    return df_contact
+
+
+def getCreneauDataFrame(csv_filename: str) -> pd.DataFrame:
+    df_creneau = pd.read_csv(csv_filename)
+    df_creneau.columns = ["nom", "lieu", "description", "responsable"]
+    return df_creneau
+
+
+def getPlanningDataFrame(csv_filename, starting_date, skip_column=3):
+    list_creneau = []
+    with io.open(csv_filename, "r", encoding="utf-8") as f:
+        datas = [split_csv_row(s.replace("\n", "")) for s in f.readlines()]
+
+    def getDate(day: str) -> datetime.datetime:
+        s = day.lower()
+        if s.startswith("mercredi"):
+            return starting_date + datetime.timedelta(days=-3)
+        elif s.startswith("jeudi"):
+            return starting_date + datetime.timedelta(days=-2)
+        elif s.startswith("vendredi"):
+            return starting_date + datetime.timedelta(days=-1)
+        elif s.startswith("samedi"):
+            return starting_date
+        elif s.startswith("dimanche"):
+            return starting_date + datetime.timedelta(days=1)
+        elif s.startswith("lundi"):
+            return starting_date + datetime.timedelta(days=2)
+        raise KeyError("This day is not valid : " + s)
+
+    def getTime(time_str: str) -> datetime.timedelta:
+        l = time_str.split("h")
+        hours = int(l[0])
+        if hours < 5:
+            hours += 24
+        if len(l) > 1 and l[1] != "":
+            return datetime.timedelta(hours=hours, minutes=int(l[1]))
+        else:
+            return datetime.timedelta(hours=hours)
+
+    def getStartEnd(time_str: str) -> tuple:
+        l = time_str.split("-")
+        if len(l) == 2:
+            return getTime(l[0]), getTime(l[1])
+        else:
+            start = getTime(time_str.split("+")[0])
+            return start, start + datetime.timedelta(hours=1)
+
+    # Parse headers
+    headers = datas[skip_column : skip_column + 2]
+    days_str = headers[0]
+    hours = headers[1]
+    column_to_dates: dict[int, dict[str, datetime.datetime]] = {}
+    assert len(hours) == len(hours)
+    for i in range(skip_column, len(days_str)):
+        day = getDate(days_str[i])
+        start, end = getStartEnd(hours[i])
+        column_to_dates[i] = {"start": day + start, "end": day + end}
+
+    list_creneau: list[dict] = []
+    for i in range(5, len(datas)):
+        row = datas[i]
+        current_benevole_name = ""
+        current_time: dict[str, datetime.datetime] = {}
+        for j in range(skip_column, len(row)):
+            if (current_benevole_name != "") and (row[j] == "" or row[j] != current_benevole_name):
+                new_creneau = {
+                    "id": uuid4(),
+                    "description_id": row[0],
+                    "nom": row[1],
+                    "benevole_nom": current_benevole_name,
+                    "ligne": i + 1,
+                    **current_time,
+                }
+                list_creneau.append(new_creneau)
+                current_benevole_name = ""
+                current_time = {}
+            if row[j] != "":
+                current_benevole_name = row[j]
+                if len(current_time.keys()) == 0:
+                    current_time = column_to_dates[j].copy()
+                else:
+                    current_time["end"] = column_to_dates[j]["end"]
+        if current_benevole_name != "":
+            new_creneau = {
+                "id": uuid4(),
+                "description_id": row[0],
+                "nom": row[1],
+                "benevole_nom": current_benevole_name,
+                "ligne": i + 1,
+                **current_time,
+            }
+            list_creneau.append(new_creneau)
+
+    print(f"{len(list_creneau)} créneaux trouvés")
+    return pd.DataFrame.from_dict(list_creneau)
+
+
+def parseGsheet(doc_uuid: str, saturday_date: datetime.datetime):
+    suffix = "_2023"
+    fname_planning = f"./planning{suffix}.csv"
+    fname_creneau = f"./creneau{suffix}.csv"
+    fname_contact = f"./benevole{suffix}.csv"
+    downloadAndSave(doc_uuid, planning_gid, fname_planning)
+    downloadAndSave(doc_uuid, creneau_gid, fname_creneau)
+    downloadAndSave(doc_uuid, benevole_gid, fname_contact)
+
+    df_contact = getContactDataFrame(fname_contact)
+    df_contact["key"] = df_contact["Prénom"] + " " + df_contact.Nom.str.slice(0, 1)
+
+    df_creneau = getCreneauDataFrame(fname_creneau)
+    df_planning = getPlanningDataFrame(fname_planning, saturday_date)
+
+    os.remove(fname_planning)
+    os.remove(fname_creneau)
+    os.remove(fname_contact)
+    return df_contact, df_creneau, df_planning

app/schemas/requests.py

@@ -46,6 +46,8 @@ class ProjectRequest(BaseRequest):
 
 class ProjectImportGsheetRequest(BaseRequest):
     sheet_url: str
+    satursday_date: datetime.datetime
+    erase_data: Optional[bool] = Field(default=False)
 
 
 class ProjectSMSBatchRequest(BaseRequest):
@@ -57,11 +59,10 @@ class ProjectSMSBatchRequest(BaseRequest):
      - {description} slot.description
      - {debut} slot.starting_time
      - {fin} slot.ending_ting
+     - {respo} slot.responsible_contact
      - {prenom} volunteer.name
      - {nom} volunteer.surname""",
-        examples=[
-            "Bonjour {prenom},\nTon créneau {titre} commence à {debut}.\nla com bénévole"
-        ],
+        examples=["Bonjour {prenom},\nTon créneau {titre} commence à {debut}.\nla com bénévole"],
     )
     delta_t: int = Field(
         default=10,

pyproject.toml

@@ -13,6 +13,8 @@ sqlalchemy = "^2.0.1"
 alembic = "^1.9.2"
 passlib = {extras = ["bcrypt"], version = "^1.7.4"}
 pydantic = {extras = ["dotenv", "email"], version = "^1.10.4"}
+pandas = "2.1.0"
+requests = "2.31.0"
 
 [tool.poetry.group.dev.dependencies]
 autoflake = "^2.0.1"

requirements.txt

@@ -23,3 +23,5 @@ sniffio==1.3.0 ; python_version >= "3.11" and python_version < "4.0"
 sqlalchemy==2.0.1 ; python_version >= "3.11" and python_version < "4.0"
 starlette==0.22.0 ; python_version >= "3.11" and python_version < "4.0"
 typing-extensions==4.4.0 ; python_version >= "3.11" and python_version < "4.0"
+pandas==2.1.0 ; python_version >= "3.11" and python_version < "4.0"
+requests==2.31.0 ; python_version >= "3.11" and python_version < "4.0"