Add data refresh capability for original tables with automatic derived table regeneration

.gitignore

@@ -5,6 +5,7 @@
 .DS_Store
 build/
 dist/
+*.egg-info/
 
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.

py-src/data_formulator/tables_routes.py

@@ -841,4 +841,111 @@ def data_loader_ingest_data_from_query():
         return jsonify({
             "status": "error", 
             "message": safe_msg
+        }), status_code
+
+
+@tables_bp.route('/refresh-derived-data', methods=['POST'])
+def refresh_derived_data():
+    """Refresh derived data by re-executing Python code on updated base table"""
+    try:
+        from data_formulator.py_sandbox import run_transform_in_sandbox2020
+
+        data = request.get_json()
+
+        # Get updated base table data and transformation info
+        updated_table = data.get('updated_table')  # {name, rows, columns}
+        derived_tables = data.get('derived_tables', [])  # [{id, code, source_tables: [names]}]
+
+        if not updated_table:
+            return jsonify({"status": "error", "message": "No updated table provided"}), 400
+
+        if not derived_tables:
+            return jsonify({"status": "error", "message": "No derived tables to refresh"}), 400
+
+        # Validate updated table has expected structure
+        updated_table_name = updated_table['name']
+        updated_columns = set(updated_table['columns'])
+
+        # Verify columns match by checking against database schema
+        with db_manager.connection(session['session_id']) as db:
+            try:
+                existing_columns = [col[0] for col in db.execute(f"DESCRIBE {updated_table_name}").fetchall()]
+                existing_columns_set = set(existing_columns)
+
+                # Validate that all existing columns are present in updated data
+                if not existing_columns_set.issubset(updated_columns):
+                    missing = existing_columns_set - updated_columns
+                    return jsonify({
+                        "status": "error",
+                        "message": f"Updated data is missing required columns: {', '.join(missing)}"
+                    }), 400
+            except Exception as e:
+                logger.warning(f"Could not validate columns for {updated_table_name}: {str(e)}")
+
+        results = []
+
+        # Process each derived table
+        for derived_info in derived_tables:
+            try:
+                code = derived_info['code']
+                source_table_names = derived_info['source_tables']
+                derived_table_id = derived_info['id']
+
+                # Prepare input dataframes
+                df_list = []
+
+                for source_name in source_table_names:
+                    if source_name == updated_table_name:
+                        # Use the updated data
+                        df = pd.DataFrame(updated_table['rows'])
+                    else:
+                        # Fetch from database
+                        with db_manager.connection(session['session_id']) as db:
+                            result = db.execute(f"SELECT * FROM {source_name}").fetchdf()
+                            df = result
+
+                    df_list.append(df)
+
+                # Execute the transformation code in subprocess for safety
+                exec_result = run_transform_in_sandbox2020(code, df_list, exec_python_in_subprocess=True)
+
+                if exec_result['status'] == 'ok':
+                    output_df = exec_result['content']
+
+                    # Convert to records format efficiently
+                    rows = output_df.to_dict(orient='records')
+                    columns = list(output_df.columns)
+
+                    results.append({
+                        'id': derived_table_id,
+                        'status': 'success',
+                        'rows': rows,
+                        'columns': columns
+                    })
+                else:
+                    results.append({
+                        'id': derived_table_id,
+                        'status': 'error',
+                        'message': exec_result['content']
+                    })
+
+            except Exception as e:
+                logger.error(f"Error refreshing derived table {derived_info.get('id')}: {str(e)}")
+                results.append({
+                    'id': derived_info.get('id'),
+                    'status': 'error',
+                    'message': str(e)
+                })
+
+        return jsonify({
+            "status": "success",
+            "results": results
+        })
+
+    except Exception as e:
+        logger.error(f"Error refreshing derived data: {str(e)}")
+        safe_msg, status_code = sanitize_db_error_message(e)
+        return jsonify({
+            "status": "error",
+            "message": safe_msg
         }), status_code

src/app/dfSlice.tsx

@@ -475,6 +475,26 @@ export const dataFormulatorSlice = createSlice({
             let attachedMetadata = action.payload.attachedMetadata;
             state.tables = state.tables.map(t => t.id == tableId ? {...t, attachedMetadata} : t);
         },
+        updateTableRows: (state, action: PayloadAction<{tableId: string, rows: any[]}>) => {
+            let tableId = action.payload.tableId;
+            let rows = action.payload.rows;
+            state.tables = state.tables.map(t => {
+                if (t.id == tableId) {
+                    // Update rows while preserving other table properties
+                    return {...t, rows};
+                }
+                return t;
+            });
+
+            // Update concept shelf items for this table if columns changed
+            let table = state.tables.find(t => t.id == tableId);
+            if (table) {
+                // Remove old field items for this table
+                state.conceptShelfItems = state.conceptShelfItems.filter(f => f.tableRef != tableId);
+                // Add new field items
+                state.conceptShelfItems = [...state.conceptShelfItems, ...getDataFieldItems(table)];
+            }
+        },
         extendTableWithNewFields: (state, action: PayloadAction<{tableId: string, columnName: string, values: any[], previousName: string | undefined, parentIDs: string[]}>) => {
             // extend the existing extTable with new columns from the new table
             let newValues = action.payload.values;