Revised NLU pipeline

.env.example

@@ -1,2 +1,3 @@
 HF_TOKEN=YOUR_SECRET_TOKEN_HERE
 HUB_MODEL_ID=YOUR_HUGGING_FACE_MODEL_ID_HERE # e.g., username/schedulebot-nlu-engine
+HUB_DS_ID=andreaceto/hasd # if you want to use my dataset

.gitignore

@@ -212,3 +212,4 @@ __marimo__/
 
 # Models
 models/
+logs/

data/appointment_id_generator.py

@@ -0,0 +1,45 @@
+import pandas as pd
+import random
+import string
+import os
+
+
+def generate_appointment_id():
+    """
+    Generates a random appointment ID in the format #123456.
+    """
+    numbers = "".join(random.choices(string.digits, k=6))
+    return f"#{numbers}"
+
+
+def create_id_file(num_to_generate, output_path):
+    """
+    Generates a specified number of IDs and saves them to a CSV file.
+    """
+    print(f"Generating {num_to_generate} appointment IDs...")
+
+    # Create a list of dictionaries
+    id_log = [
+        {"text": generate_appointment_id(), "entity": "appointment_id"}
+        for _ in range(num_to_generate)
+    ]
+
+    # Convert to DataFrame
+    ids_df = pd.DataFrame(id_log)
+
+    # Ensure the directory exists
+    output_dir = os.path.dirname(output_path)
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Save to CSV
+    ids_df.to_csv(output_path, index=False)
+    print(f"✅ Successfully saved IDs to '{output_path}'")
+
+
+# This part runs when you execute the script directly
+if __name__ == "__main__":
+    # --- Configuration ---
+    NUM_IDS_TO_GENERATE = 150
+    OUTPUT_FILE = os.path.join("data", "raw", "entities", "appointment_id.csv")
+
+    create_id_file(NUM_IDS_TO_GENERATE, OUTPUT_FILE)

data/processed/hasd_processed/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train", "validation", "test"]}

data/processed/hasd_processed/test/dataset_info.json

@@ -0,0 +1,97 @@
+{
+  "builder_name": "json",
+  "citation": "",
+  "config_name": "default",
+  "dataset_name": "json",
+  "dataset_size": 249489,
+  "description": "",
+  "download_checksums": {
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/train.jsonl": {
+      "num_bytes": 248004,
+      "checksum": null
+    },
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/validation.jsonl": {
+      "num_bytes": 51624,
+      "checksum": null
+    },
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/test.jsonl": {
+      "num_bytes": 52346,
+      "checksum": null
+    }
+  },
+  "download_size": 351974,
+  "features": {
+    "input_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "intent_label": {
+      "names": [
+        "bye",
+        "cancel",
+        "greeting",
+        "negative_reply",
+        "oos",
+        "positive_reply",
+        "query_avail",
+        "reschedule",
+        "schedule"
+      ],
+      "_type": "ClassLabel"
+    },
+    "labels": {
+      "feature": {
+        "names": [
+          "O",
+          "B-appointment_id",
+          "I-appointment_id",
+          "B-appointment_type",
+          "I-appointment_type",
+          "B-practitioner_name",
+          "I-practitioner_name"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "List"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 601463,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 175986,
+      "num_examples": 2045,
+      "dataset_name": "json"
+    },
+    "validation": {
+      "name": "validation",
+      "num_bytes": 36476,
+      "num_examples": 438,
+      "dataset_name": "json"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 37027,
+      "num_examples": 439,
+      "dataset_name": "json"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

data/processed/hasd_processed/test/state.json

@@ -0,0 +1,18 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "8a7c1f454b53d81c",
+  "_format_columns": [
+    "input_ids",
+    "attention_mask",
+    "intent_label",
+    "labels"
+  ],
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": "test"
+}

data/processed/hasd_processed/train/dataset_info.json

@@ -0,0 +1,97 @@
+{
+  "builder_name": "json",
+  "citation": "",
+  "config_name": "default",
+  "dataset_name": "json",
+  "dataset_size": 249489,
+  "description": "",
+  "download_checksums": {
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/train.jsonl": {
+      "num_bytes": 248004,
+      "checksum": null
+    },
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/validation.jsonl": {
+      "num_bytes": 51624,
+      "checksum": null
+    },
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/test.jsonl": {
+      "num_bytes": 52346,
+      "checksum": null
+    }
+  },
+  "download_size": 351974,
+  "features": {
+    "input_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "intent_label": {
+      "names": [
+        "bye",
+        "cancel",
+        "greeting",
+        "negative_reply",
+        "oos",
+        "positive_reply",
+        "query_avail",
+        "reschedule",
+        "schedule"
+      ],
+      "_type": "ClassLabel"
+    },
+    "labels": {
+      "feature": {
+        "names": [
+          "O",
+          "B-appointment_id",
+          "I-appointment_id",
+          "B-appointment_type",
+          "I-appointment_type",
+          "B-practitioner_name",
+          "I-practitioner_name"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "List"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 601463,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 175986,
+      "num_examples": 2045,
+      "dataset_name": "json"
+    },
+    "validation": {
+      "name": "validation",
+      "num_bytes": 36476,
+      "num_examples": 438,
+      "dataset_name": "json"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 37027,
+      "num_examples": 439,
+      "dataset_name": "json"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

data/processed/hasd_processed/train/state.json

@@ -0,0 +1,18 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "bbf42d4f1a796c18",
+  "_format_columns": [
+    "input_ids",
+    "attention_mask",
+    "intent_label",
+    "labels"
+  ],
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": "train"
+}

data/processed/hasd_processed/validation/dataset_info.json

@@ -0,0 +1,97 @@
+{
+  "builder_name": "json",
+  "citation": "",
+  "config_name": "default",
+  "dataset_name": "json",
+  "dataset_size": 249489,
+  "description": "",
+  "download_checksums": {
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/train.jsonl": {
+      "num_bytes": 248004,
+      "checksum": null
+    },
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/validation.jsonl": {
+      "num_bytes": 51624,
+      "checksum": null
+    },
+    "C:/Users/aceto/Documents/GitHub/schedulebot-plus/data/raw/test.jsonl": {
+      "num_bytes": 52346,
+      "checksum": null
+    }
+  },
+  "download_size": 351974,
+  "features": {
+    "input_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "intent_label": {
+      "names": [
+        "bye",
+        "cancel",
+        "greeting",
+        "negative_reply",
+        "oos",
+        "positive_reply",
+        "query_avail",
+        "reschedule",
+        "schedule"
+      ],
+      "_type": "ClassLabel"
+    },
+    "labels": {
+      "feature": {
+        "names": [
+          "O",
+          "B-appointment_id",
+          "I-appointment_id",
+          "B-appointment_type",
+          "I-appointment_type",
+          "B-practitioner_name",
+          "I-practitioner_name"
+        ],
+        "_type": "ClassLabel"
+      },
+      "_type": "List"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 601463,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 175986,
+      "num_examples": 2045,
+      "dataset_name": "json"
+    },
+    "validation": {
+      "name": "validation",
+      "num_bytes": 36476,
+      "num_examples": 438,
+      "dataset_name": "json"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 37027,
+      "num_examples": 439,
+      "dataset_name": "json"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}