diff --git a/.gitignore b/.gitignore index df0ef7c..8ca846c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .DS_Store build/ dist/ +*.egg-info/ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/py-src/data_formulator/tables_routes.py b/py-src/data_formulator/tables_routes.py index 74f31f2..0ffaad3 100644 --- a/py-src/data_formulator/tables_routes.py +++ b/py-src/data_formulator/tables_routes.py @@ -841,4 +841,111 @@ def data_loader_ingest_data_from_query(): return jsonify({ "status": "error", "message": safe_msg + }), status_code + + +@tables_bp.route('/refresh-derived-data', methods=['POST']) +def refresh_derived_data(): + """Refresh derived data by re-executing Python code on updated base table""" + try: + from data_formulator.py_sandbox import run_transform_in_sandbox2020 + + data = request.get_json() + + # Get updated base table data and transformation info + updated_table = data.get('updated_table') # {name, rows, columns} + derived_tables = data.get('derived_tables', []) # [{id, code, source_tables: [names]}] + + if not updated_table: + return jsonify({"status": "error", "message": "No updated table provided"}), 400 + + if not derived_tables: + return jsonify({"status": "error", "message": "No derived tables to refresh"}), 400 + + # Validate updated table has expected structure + updated_table_name = updated_table['name'] + updated_columns = set(updated_table['columns']) + + # Verify columns match by checking against database schema + with db_manager.connection(session['session_id']) as db: + try: + existing_columns = [col[0] for col in db.execute(f"DESCRIBE {updated_table_name}").fetchall()] + existing_columns_set = set(existing_columns) + + # Validate that all existing columns are present in updated data + if not existing_columns_set.issubset(updated_columns): + missing = existing_columns_set - updated_columns + return jsonify({ + "status": "error", + "message": f"Updated data is missing required columns: {', '.join(missing)}" + }), 400 + except Exception as e: + logger.warning(f"Could not validate columns for {updated_table_name}: {str(e)}") + + results = [] + + # Process each derived table + for derived_info in derived_tables: + try: + code = derived_info['code'] + source_table_names = derived_info['source_tables'] + derived_table_id = derived_info['id'] + + # Prepare input dataframes + df_list = [] + + for source_name in source_table_names: + if source_name == updated_table_name: + # Use the updated data + df = pd.DataFrame(updated_table['rows']) + else: + # Fetch from database + with db_manager.connection(session['session_id']) as db: + result = db.execute(f"SELECT * FROM {source_name}").fetchdf() + df = result + + df_list.append(df) + + # Execute the transformation code in subprocess for safety + exec_result = run_transform_in_sandbox2020(code, df_list, exec_python_in_subprocess=True) + + if exec_result['status'] == 'ok': + output_df = exec_result['content'] + + # Convert to records format efficiently + rows = output_df.to_dict(orient='records') + columns = list(output_df.columns) + + results.append({ + 'id': derived_table_id, + 'status': 'success', + 'rows': rows, + 'columns': columns + }) + else: + results.append({ + 'id': derived_table_id, + 'status': 'error', + 'message': exec_result['content'] + }) + + except Exception as e: + logger.error(f"Error refreshing derived table {derived_info.get('id')}: {str(e)}") + results.append({ + 'id': derived_info.get('id'), + 'status': 'error', + 'message': str(e) + }) + + return jsonify({ + "status": "success", + "results": results + }) + + except Exception as e: + logger.error(f"Error refreshing derived data: {str(e)}") + safe_msg, status_code = sanitize_db_error_message(e) + return jsonify({ + "status": "error", + "message": safe_msg }), status_code \ No newline at end of file diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index 393d134..16a9cb4 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -475,6 +475,26 @@ export const dataFormulatorSlice = createSlice({ let attachedMetadata = action.payload.attachedMetadata; state.tables = state.tables.map(t => t.id == tableId ? {...t, attachedMetadata} : t); }, + updateTableRows: (state, action: PayloadAction<{tableId: string, rows: any[]}>) => { + let tableId = action.payload.tableId; + let rows = action.payload.rows; + state.tables = state.tables.map(t => { + if (t.id == tableId) { + // Update rows while preserving other table properties + return {...t, rows}; + } + return t; + }); + + // Update concept shelf items for this table if columns changed + let table = state.tables.find(t => t.id == tableId); + if (table) { + // Remove old field items for this table + state.conceptShelfItems = state.conceptShelfItems.filter(f => f.tableRef != tableId); + // Add new field items + state.conceptShelfItems = [...state.conceptShelfItems, ...getDataFieldItems(table)]; + } + }, extendTableWithNewFields: (state, action: PayloadAction<{tableId: string, columnName: string, values: any[], previousName: string | undefined, parentIDs: string[]}>) => { // extend the existing extTable with new columns from the new table let newValues = action.payload.values; diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index d873edb..d199972 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -22,7 +22,10 @@ import { Popper, Paper, ClickAwayListener, - Badge + Badge, + Menu, + MenuItem, + ListItemText } from '@mui/material'; import { VegaLite } from 'react-vega' @@ -46,6 +49,8 @@ import CloseIcon from '@mui/icons-material/Close'; import HelpOutlineIcon from '@mui/icons-material/HelpOutline'; import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; import CancelOutlinedIcon from '@mui/icons-material/CancelOutlined'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; import _ from 'lodash'; import { getChartTemplate } from '../components/ChartTemplates'; @@ -196,6 +201,195 @@ const MetadataPopup = memo<{ ); }); +// Refresh Data Dialog Component +const RefreshDataDialog = memo<{ + open: boolean; + anchorEl: HTMLElement | null; + onClose: () => void; + onRefresh: (file: File | null, rawData: string) => void; + tableName: string; + tableColumns: string[]; +}>(({ open, anchorEl, onClose, onRefresh, tableName, tableColumns }) => { + const [uploadMode, setUploadMode] = useState<'file' | 'raw'>('file'); + const [selectedFile, setSelectedFile] = useState(null); + const [rawData, setRawData] = useState(''); + const [error, setError] = useState(''); + + useEffect(() => { + if (!open) { + setSelectedFile(null); + setRawData(''); + setError(''); + setUploadMode('file'); + } + }, [open]); + + const validateData = (data: any[]) => { + if (!Array.isArray(data) || data.length === 0) { + return 'Data must be a non-empty array of objects'; + } + + const expectedColumns = new Set(tableColumns); + + // Validate all objects have the same columns + for (const row of data) { + if (typeof row !== 'object' || row === null) { + return 'All data elements must be objects'; + } + + const rowColumns = new Set(Object.keys(row)); + + // Check if all expected columns are present + for (const col of expectedColumns) { + if (!rowColumns.has(col)) { + return `Missing required column: ${col}`; + } + } + } + + return null; + }; + + const handleFileChange = (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (file) { + setSelectedFile(file); + setError(''); + } + }; + + const handleRefresh = async () => { + try { + if (uploadMode === 'file' && selectedFile) { + // Validate file extension + const ext = selectedFile.name.split('.').pop()?.toLowerCase(); + if (!['csv', 'xlsx', 'xls', 'json'].includes(ext || '')) { + setError('Unsupported file format. Please use CSV, XLSX, or JSON.'); + return; + } + + onRefresh(selectedFile, ''); + onClose(); + } else if (uploadMode === 'raw' && rawData) { + // Validate JSON format + try { + const parsed = JSON.parse(rawData); + const validationError = validateData(parsed); + if (validationError) { + setError(validationError); + return; + } + onRefresh(null, rawData); + onClose(); + } catch (e) { + setError('Invalid JSON format'); + } + } else { + setError('Please provide data to refresh'); + } + } catch (e) { + setError('Error processing data'); + } + }; + + return ( + + + + + Refresh data for {tableName} + + + Upload new data with the same column names: {tableColumns.join(', ')} + + + + + + + + {uploadMode === 'file' ? ( + + + {selectedFile && ( + + Selected: {selectedFile.name} + + )} + + ) : ( + { + setRawData(e.target.value); + setError(''); + }} + sx={{ my: 1, '& .MuiInputBase-input': { fontSize: 12 } }} + /> + )} + + {error && ( + + {error} + + )} + + + + + + + + + ); +}); + // Agent Status Box Component const AgentStatusBox = memo<{ tableId: string; @@ -490,6 +684,16 @@ let SingleThreadGroupView: FC<{ const [selectedTableForMetadata, setSelectedTableForMetadata] = useState(null); const [metadataAnchorEl, setMetadataAnchorEl] = useState(null); + // Refresh data popup state + const [refreshDataPopupOpen, setRefreshDataPopupOpen] = useState(false); + const [selectedTableForRefresh, setSelectedTableForRefresh] = useState(null); + const [refreshDataAnchorEl, setRefreshDataAnchorEl] = useState(null); + + // Menu state for actions + const [menuAnchorEl, setMenuAnchorEl] = useState(null); + const [menuOpen, setMenuOpen] = useState(false); + const [menuTableId, setMenuTableId] = useState(null); + let handleUpdateTableDisplayId = (tableId: string, displayId: string) => { dispatch(dfActions.updateTableDisplayId({ @@ -519,6 +723,124 @@ let SingleThreadGroupView: FC<{ } }; + const handleOpenRefreshDataPopup = (table: DictTable, anchorEl: HTMLElement) => { + setSelectedTableForRefresh(table); + setRefreshDataAnchorEl(anchorEl); + setRefreshDataPopupOpen(true); + }; + + const handleCloseRefreshDataPopup = () => { + setRefreshDataPopupOpen(false); + setSelectedTableForRefresh(null); + setRefreshDataAnchorEl(null); + }; + + const handleRefreshData = async (file: File | null, rawData: string) => { + if (!selectedTableForRefresh) return; + + try { + const formData = new FormData(); + formData.append('table_name', selectedTableForRefresh.id); + + if (file) { + formData.append('file', file); + } else if (rawData) { + formData.append('raw_data', rawData); + } + + // First, replace the table data in the database + const replaceResponse = await fetch('/api/tables/create-table', { + method: 'POST', + body: formData + }); + + const replaceResult = await replaceResponse.json(); + + if (replaceResult.status !== 'success') { + throw new Error(replaceResult.message || 'Failed to replace table data'); + } + + // Get the updated table data from server + const tableResponse = await fetch(`/api/tables/get-table?table_name=${selectedTableForRefresh.id}`); + const tableResult = await tableResponse.json(); + + if (tableResult.status !== 'success') { + throw new Error('Failed to fetch updated table data'); + } + + // Update the base table in Redux + dispatch(dfActions.updateTableRows({ + tableId: selectedTableForRefresh.id, + rows: tableResult.rows + })); + + // Find all derived tables that depend on this table + const derivedTables = tables.filter(t => { + if (!t.derive?.source) return false; + // Check if source is an array or string and handle accordingly + const sources = Array.isArray(t.derive.source) ? t.derive.source : [t.derive.source]; + return sources.includes(selectedTableForRefresh.id); + }); + + if (derivedTables.length > 0) { + // Call the refresh-derived-data endpoint + const refreshPayload = { + updated_table: { + name: selectedTableForRefresh.id, + rows: tableResult.rows, + columns: tableResult.columns + }, + derived_tables: derivedTables.map(dt => ({ + id: dt.id, + code: dt.derive?.code || '', + source_tables: dt.derive?.source || [] + })) + }; + + const refreshResponse = await fetch('/api/tables/refresh-derived-data', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify(refreshPayload) + }); + + const refreshResult = await refreshResponse.json(); + + if (refreshResult.status === 'success') { + // Update Redux state with refreshed derived tables + refreshResult.results.forEach((result: any) => { + if (result.status === 'success') { + dispatch(dfActions.updateTableRows({ + tableId: result.id, + rows: result.rows + })); + } else { + console.error(`Failed to refresh table ${result.id}:`, result.message); + } + }); + } + } + + } catch (error) { + console.error('Error refreshing data:', error); + alert(`Error refreshing data: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + }; + + const handleMenuOpen = (event: React.MouseEvent, tableId: string) => { + event.stopPropagation(); + setMenuAnchorEl(event.currentTarget); + setMenuTableId(tableId); + setMenuOpen(true); + }; + + const handleMenuClose = () => { + setMenuOpen(false); + setMenuAnchorEl(null); + setMenuTableId(null); + }; + let buildTriggerCard = (trigger: Trigger) => { let selectedClassName = trigger.chart?.id == focusedChartId ? 'selected-card' : ''; @@ -673,40 +995,46 @@ let SingleThreadGroupView: FC<{ - {table?.derive == undefined && - { - event.stopPropagation(); - handleOpenMetadataPopup(table!, event.currentTarget); - }} - > - - - } + {table?.derive == undefined && ( + + handleMenuOpen(event, tableId)} + > + + + + )} - {tableDeleteEnabled && - { - event.stopPropagation(); - dispatch(dfActions.deleteTable(tableId)); - }} - > - - - } + {table?.derive !== undefined && tableDeleteEnabled && ( + + { + event.stopPropagation(); + dispatch(dfActions.deleteTable(tableId)); + }} + > + + + + )} + + {/* Menu for original table actions */} + + { + event.stopPropagation(); + handleOpenMetadataPopup(table!, menuAnchorEl!); + }}> + + + + + + { + event.stopPropagation(); + handleOpenRefreshDataPopup(table!, menuAnchorEl!); + }}> + + + + + + {tableDeleteEnabled && ( + { + event.stopPropagation(); + dispatch(dfActions.deleteTable(tableId)); + }}> + + + + + + )} + @@ -888,6 +1265,14 @@ let SingleThreadGroupView: FC<{ initialValue={selectedTableForMetadata?.attachedMetadata || ''} tableName={selectedTableForMetadata?.displayId || selectedTableForMetadata?.id || ''} /> + } diff --git a/yarn.lock b/yarn.lock index f1d9bfb..0baef35 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4801,10 +4801,10 @@ uuid@^8.3.0: resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== -validator@^13.15.22: - version "13.15.22" - resolved "https://registry.yarnpkg.com/validator/-/validator-13.15.22.tgz#5f847cf4a799107e5716fc87e5cf2a337a71eb14" - integrity sha512-uT/YQjiyLJP7HSrv/dPZqK9L28xf8hsNca01HSz1dfmI0DgMfjopp1rO/z13NeGF1tVystF0Ejx3y4rUKPw+bQ== +validator@^13.15.20: + version "13.15.26" + resolved "https://registry.yarnpkg.com/validator/-/validator-13.15.26.tgz#36c3deeab30e97806a658728a155c66fcaa5b944" + integrity sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA== vega-canvas@^2.0.0: version "2.0.0"