clean up predict pipeline 1

predict_pipeline/check_python_version.py

@@ -1,11 +0,0 @@
-# from tools import db_helpers
-import sys
-
-
-def main():
-    print(sys.version)
-    # db_helpers.add_columns_to_table()    
-
-
-if __name__ == "__main__":
-    main()

predict_pipeline/feature_extraction.py

@@ -1,9 +0,0 @@
-import sqlite3
-
-def main():
-
-    return 0
-
-
-if __name__ == "__main__":
-    main()

predict_pipeline/fill_db.ipynb

@@ -1,5 +1,13 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "fb68b447",
+   "metadata": {},
+   "source": [
+    "## Database creation and filling (for live system) "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -21,8 +29,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "database_path = Path(r\"/home/edgekit/MSY_FS/databases/rawdata.sqlite\")\n",
+    "# TODO: set paths and table name\n",
-    "parquet_path = Path(r\"/home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/files_for_testing/both_mod_0000.parquet\")"
+    "database_path = Path(r\"database.sqlite\") # this path references an empty, but already created sqlite file\n",
+    "parquet_path = Path(r\"...parquet\") # this path leads to the data that should be used to fill the databse\n",
+    "table_name =  \"XXX\" # name of the new table"
    ]
   },
   {
@@ -56,6 +66,14 @@
     "con, cursor = db_helpers.connect_db(database_path)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "7007c68f",
+   "metadata": {},
+   "source": [
+    "Select a subset to insert into database "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -69,6 +87,14 @@
     "df_first_100.insert(0, '_Id', df_first_100.index + 1)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "92171186",
+   "metadata": {},
+   "source": [
+    "Type conversion"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -88,6 +114,14 @@
     "    return \"TEXT\"\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "45af9956",
+   "metadata": {},
+   "source": [
+    "Define constraints and primary key"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -109,6 +143,14 @@
     "}\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "133e92ee",
+   "metadata": {},
+   "source": [
+    "Create the table"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -119,7 +161,7 @@
     "sql = db_helpers.create_table(\n",
     "    conn=con,\n",
     "    cursor=cursor,\n",
-    "    table_name=\"rawdata\",\n",
+    "    table_name=table_name,\n",
     "    columns=columns,\n",
     "    constraints=constraints,\n",
     "    primary_key=primary_key,\n",
@@ -150,7 +192,7 @@
     "db_helpers.insert_rows_into_table(\n",
     "    conn=con,\n",
     "    cursor=cursor,\n",
-    "    table_name=\"rawdata\",\n",
+    "    table_name=table_name,\n",
     "    columns=columns_to_insert,\n",
     "    commit=True\n",
     ")\n"
@@ -163,7 +205,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "a = db_helpers.get_data_from_table(conn=con, table_name='rawdata',columns_list=['*'])"
+    "request = db_helpers.get_data_from_table(conn=con, table_name='rawdata',columns_list=['*'])"
    ]
   },
   {
@@ -173,7 +215,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "a.head()"
+    "request.head()"
    ]
   },
   {
@@ -189,7 +231,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "MSY_FS_env",
+   "display_name": "310",
    "language": "python",
    "name": "python3"
   },
@@ -203,7 +245,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.12"
+   "version": "3.10.19"
   }
  },
  "nbformat": 4,