diff --git a/VERSION b/VERSION index ee672d8..ed21137 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.9.1 \ No newline at end of file +1.10.0 \ No newline at end of file diff --git a/core/utils/date_utils.py b/core/utils/date_utils.py index 56bfa33..026d434 100644 --- a/core/utils/date_utils.py +++ b/core/utils/date_utils.py @@ -36,6 +36,11 @@ def get_date_range_str(from_date_str: str = None, until_date_str: str = None, da """ Get the date range to be used in the queries. + If both from_date_str and until_date_str are provided, they will be used. + If only one is provided, it will be used as the start or end date, and the other will be calculated based on a 7-day range. + If neither is provided, the function will default to the last 7 days from today. + If days_to_go_back is provided, it will override the from_date_str and until_date_str. + Args: from_date_str: Date to start the query. until_date_str: Date to end the query. diff --git a/log_manager/tasks.py b/log_manager/tasks.py index 5a04204..76ac317 100644 --- a/log_manager/tasks.py +++ b/log_manager/tasks.py @@ -169,7 +169,7 @@ def _fetch_validation_parameters(collection, default_buffer_size=0.1, default_sa @celery_app.task(bind=True, name=_('Check missing log files')) -def task_check_missing_logs_for_date_range(self, collections=[], from_date=None, until_date=None, user_id=None, username=None): +def task_check_missing_logs_for_date_range(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None): """ Task to check for missing log files in the defined date range. @@ -177,6 +177,7 @@ def task_check_missing_logs_for_date_range(self, collections=[], from_date=None, collections (list, optional): List of collection acronyms. Defaults to []. from_date (str, optional): The start date for log discovery in YYYY-MM-DD format. Defaults to None. until_date (str, optional): The end date for log discovery in YYYY-MM-DD format. Defaults to None. + days_to_go_back (int, optional): The number of days to go back from today for log discovery. Defaults to None. user_id (int, optional): The ID of the user initiating the task. Defaults to None. username (str, optional): The username of the user initiating the task. Defaults to None. @@ -186,9 +187,11 @@ def task_check_missing_logs_for_date_range(self, collections=[], from_date=None, """ user = _get_user(self.request, username=username, user_id=user_id) + from_date_str, until_date_str = date_utils.get_date_range_str(from_date, until_date, days_to_go_back) + for col in collections or Collection.acron3_list(): collection = Collection.objects.get(acron3=col) - for date in date_utils.get_date_objs_from_date_range(from_date, until_date): + for date in date_utils.get_date_objs_from_date_range(from_date_str, until_date_str): logging.info(f'Checking missings logs for collection {col} and date {date}') _check_missing_logs_for_date(user, collection, date) @@ -214,8 +217,8 @@ def _check_missing_logs_for_date(user, collection, date): @celery_app.task(bind=True, name=_('Generate log files count report')) -def task_log_files_count_status_report(self, collections=[], from_date=None, until_date=None, user_id=None, username=None): - from_date, until_date = date_utils.get_date_range_str(from_date, until_date) +def task_log_files_count_status_report(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None): + from_date, until_date = date_utils.get_date_range_str(from_date, until_date, days_to_go_back) from_date_obj = date_utils.get_date_obj(from_date) until_date_obj = date_utils.get_date_obj(until_date) diff --git a/metrics/tasks.py b/metrics/tasks.py index 2e99174..2859003 100644 --- a/metrics/tasks.py +++ b/metrics/tasks.py @@ -48,7 +48,7 @@ @celery_app.task(bind=True, name=_('Compute access'), timelimit=-1) -def task_parse_logs(self, collections=[], from_date=None, until_date=None, user_id=None, username=None): +def task_parse_logs(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None): """ Parses log files associated with a given collection. @@ -56,13 +56,14 @@ def task_parse_logs(self, collections=[], from_date=None, until_date=None, user_ collections (list, optional): List of collection acronyms to parse logs for. Defaults to all collections. from_date (str, optional): Start date for log parsing in 'YYYY-MM-DD' format. Defaults to None. until_date (str, optional): End date for log parsing in 'YYYY-MM-DD' format. Defaults to None. + days_to_go_back (int, optional): Number of days to go back from the current date to parse logs. Defaults to None. user_id username Returns: None. """ - from_date, until_date = get_date_range_str(from_date, until_date) + from_date, until_date = get_date_range_str(from_date, until_date, days_to_go_back) from_date_obj = get_date_obj(from_date) until_date_obj = get_date_obj(until_date) @@ -373,7 +374,7 @@ def task_delete_documents_by_key(self, keys, values, index_name=None, user_id=No logging.error(f"Failed to delete documents with keys {keys} and values {values} from index {index_name}: {e}") -@celery_app.task(bind=True, name=_('Compute metrics'), timelimit=-1) +@celery_app.task(bind=True, name=_('Index metrics'), timelimit=-1) def task_index_documents(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None, bulk_size=5000, replace=False): """ Task to compute and index metrics for specified collections within a given date range.