Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,12 @@ jobs:
if: "!contains(github.event.head_commit.message, 'ci skip')"
strategy:
fail-fast: false
max-parallel: 1
matrix:
os:
- 'ubuntu-22.04'
- 'windows-2022'
- 'macos-13'
- 'macos-14'
python-version:
- '3.8'

Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,12 @@ jobs:
if: "!contains(github.event.head_commit.message, 'ci skip')"
strategy:
fail-fast: false
max-parallel: 1
matrix:
os:
- 'ubuntu-22.04'
- 'windows-2022'
- 'macos-13'
- 'macos-14'
python-version:
- '3.8'

Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ jobs:
if: ${{ !contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'test skip') }}
strategy:
fail-fast: false
max-parallel: 2
max-parallel: 1
matrix:
os:
- 'ubuntu-latest'
- 'windows-latest'
- 'macos-latest'
python-version:
- '3.8'
- '3.9'
- '3.10'
# - '3.9'
# - '3.10'
- '3.11'
- '3.12'
- '3.13'
# - '3.12'
# - '3.13'

steps:
- name: Get system version for Linux
Expand Down Expand Up @@ -132,7 +132,7 @@ jobs:
os:
- 'ubuntu-22.04'
- 'windows-2022'
- 'macos-13'
- 'macos-14'
python-version:
- '3.8'

Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,16 @@
from typing import Optional

import click
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
from ..operate import download_file_to_file, download_archive_as_directory, download_directory_as_directory
from ..operate.base import REPO_TYPES, RepoTypeTyping
from ..utils import get_requests_session
from ..utils import get_requests_session, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


class NoRemotePathAssignedWithDownload(ClickErrorException):
Expand Down Expand Up @@ -122,7 +126,8 @@ def download(

:raises NoRemotePathAssignedWithDownload: If no remote path in repository is assigned.
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

if tmpdir:
os.environ['TMPDIR'] = tmpdir
Expand Down
12 changes: 9 additions & 3 deletions hfutils/entry/duplicate.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,16 @@
"""

import click
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS, command_wrap
from ..operate import hf_repo_duplicate
from ..operate.base import REPO_TYPES, RepoTypeTyping
from ..utils import get_requests_session
from ..utils import get_requests_session, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


def _add_duplicate_subcommand(cli: click.Group) -> click.Group:
Expand Down Expand Up @@ -59,7 +63,9 @@ def duplicate(src_repo_id: str, dst_repo_id: str, repo_type: RepoTypeTyping, pri

:raises: Various exceptions from huggingface_hub based on operation status
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

hf_repo_duplicate(
src_repo_id=src_repo_id,
dst_repo_id=dst_repo_id,
Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/ils.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,18 @@
import click
from hbutils.scale import size_to_bytes_str
from hbutils.string import plural_word, titleize
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS
from ..index import hf_tar_get_index, hf_tar_validate
from ..operate.base import REPO_TYPES
from ..utils import get_requests_session, get_file_type, FileItemType
from ..utils import get_requests_session, get_file_type, FileItemType, HF_IS_VERSION_0_X_X
from ..utils.path import RepoTypeTyping, hf_normpath

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None

_FT_NAME_MAP = {
FileItemType.IMAGE: 'image',
FileItemType.ARCHIVE: 'archive/compressed',
Expand Down Expand Up @@ -133,7 +137,8 @@ def ls(repo_id: str, idx_repo_id: Optional[str], repo_type: RepoTypeTyping, revi

The function uses click styles to format the output for better readability in the terminal.
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

idx_info = hf_tar_get_index(
repo_id=repo_id,
Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,19 @@

import click
from hbutils.string import plural_word
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS
from ..cache import delete_detached_cache
from ..index import hf_tar_validate, tar_create_index
from ..operate import get_hf_fs, download_file_to_file, upload_directory_as_directory
from ..operate.base import REPO_TYPES, RepoTypeTyping, get_hf_client
from ..utils import tqdm, hf_fs_path, parse_hf_fs_path, TemporaryDirectory, hf_normpath, ColoredFormatter, \
get_requests_session
get_requests_session, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


def _add_index_subcommand(cli: click.Group) -> click.Group:
Expand Down Expand Up @@ -90,7 +94,8 @@ def index(repo_id: str, idx_repo_id: Optional[str], repo_type: RepoTypeTyping, r
This function is typically invoked through the CLI interface, like:
$ python script.py index -r my_repo -x my_index_repo -t dataset -R main --min_upload_interval 120
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

logger = logging.getLogger()
logger.setLevel(logging.INFO)
Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/ls.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,16 @@

import click
import tzlocal
from huggingface_hub import configure_http_backend
from huggingface_hub.hf_api import RepoFolder, RepoFile

from .base import CONTEXT_SETTINGS
from ..operate.base import REPO_TYPES, get_hf_client
from ..utils import get_requests_session, FileItemType, get_file_type
from ..utils import get_requests_session, FileItemType, get_file_type, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


class ListItem:
Expand Down Expand Up @@ -73,7 +77,8 @@ def ls(repo_id: str, repo_type: str, dir_in_repo, revision: str, show_all: bool,
:param show_detailed: Flag to indicate whether to show detailed file information.
:type show_detailed: bool
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

hf_client = get_hf_client()
items: List[ListItem] = []
Expand Down
14 changes: 11 additions & 3 deletions hfutils/entry/ls_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@
from typing import Optional

import click
from huggingface_hub import configure_http_backend
from huggingface_hub.utils import LocalTokenNotFoundError

from .base import CONTEXT_SETTINGS, ClickErrorException
from ..operate.base import REPO_TYPES, get_hf_client
from ..utils import get_requests_session
from ..utils import get_requests_session, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


class NoLocalAuthentication(ClickErrorException):
Expand Down Expand Up @@ -48,11 +52,15 @@ def ls(author: Optional[str], repo_type: str, pattern: str):
:param pattern: Pattern of the repository names.
:type pattern: str
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

hf_client = get_hf_client()
if not author:
try:
if not hf_client.token:
raise LocalTokenNotFoundError

info = hf_client.whoami()
author = author or info['name']
except LocalTokenNotFoundError:
Expand Down
10 changes: 9 additions & 1 deletion hfutils/entry/rollback.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,12 @@
from .base import CONTEXT_SETTINGS
from ..operate.base import REPO_TYPES, RepoTypeTyping
from ..repository import hf_hub_rollback
from ..utils import ColoredFormatter
from ..utils import ColoredFormatter, HF_IS_VERSION_0_X_X, get_requests_session

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


def _add_rollback_subcommand(cli: click.Group) -> click.Group:
Expand Down Expand Up @@ -43,6 +48,9 @@ def rollback(repo_id: str, repo_type: RepoTypeTyping, revision: str, rollback_to
:param revision: The revision of the repository to rollback.
:param rollback_to_commit_id: The commit ID to rollback to.
"""
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

# Set up logging
logger = logging.getLogger()
logger.setLevel(logging.INFO)
Expand Down
12 changes: 9 additions & 3 deletions hfutils/entry/squash.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,15 @@
from typing import Optional

import click
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS, command_wrap
from ..operate.base import REPO_TYPES, RepoTypeTyping, get_hf_client
from ..utils import get_requests_session, hf_fs_path
from ..utils import get_requests_session, hf_fs_path, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


def _add_squash_subcommand(cli: click.Group) -> click.Group:
Expand Down Expand Up @@ -59,7 +63,9 @@ def squash(repo_id: str, repo_type: RepoTypeTyping, revision: str, message: Opti
:param message: Optional commit message for the squash operation.
:type message: Optional[str]
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

hf_client = get_hf_client()
hf_client.super_squash_history(
repo_id=repo_id,
Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,17 @@

import click
from hbutils.string import format_tree
from huggingface_hub import configure_http_backend
from huggingface_hub.hf_api import RepoFile
from natsort import natsorted

from .base import CONTEXT_SETTINGS
from ..operate.base import REPO_TYPES, list_files_in_repository, RepoTypeTyping, get_hf_client
from ..utils import get_requests_session, hf_normpath, get_file_type, hf_fs_path, FileItemType
from ..utils import get_requests_session, hf_normpath, get_file_type, hf_fs_path, FileItemType, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


@dataclasses.dataclass
Expand Down Expand Up @@ -221,7 +225,8 @@ def tree(repo_id: str, repo_type: RepoTypeTyping, dir_in_repo, revision: str, sh
:param show_all: Whether to show hidden files.
:type show_all: bool
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

_tree = _get_tree(
repo_id=repo_id,
Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,16 @@
from typing import Optional

import click
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
from ..operate import upload_file_to_file, upload_directory_as_archive, upload_directory_as_directory
from ..operate.base import REPO_TYPES, RepoTypeTyping, get_hf_client
from ..utils import get_requests_session
from ..utils import get_requests_session, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


class NoRemotePathAssignedWithUpload(ClickErrorException):
Expand Down Expand Up @@ -123,7 +127,8 @@ def upload(repo_id: str, repo_type: RepoTypeTyping,
:raises NoRemotePathAssignedWithUpload: If no upload mode is specified.
:raises ValueError: If conflicting visibility settings are provided.
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

if not file_in_repo and not archive_in_repo and not dir_in_repo:
raise NoRemotePathAssignedWithUpload('No remote path in repository assigned.\n'
Expand Down
11 changes: 8 additions & 3 deletions hfutils/entry/warmup.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,16 @@
from typing import Optional

import click
from huggingface_hub import configure_http_backend

from .base import CONTEXT_SETTINGS, command_wrap, ClickErrorException
from ..operate import hf_warmup_file, hf_warmup_directory
from ..operate.base import REPO_TYPES, RepoTypeTyping
from ..utils import get_requests_session
from ..utils import get_requests_session, HF_IS_VERSION_0_X_X

if HF_IS_VERSION_0_X_X:
from huggingface_hub import configure_http_backend
else:
configure_http_backend = None


class NoRemotePathAssignedWithWarmup(ClickErrorException):
Expand Down Expand Up @@ -100,7 +104,8 @@ def warmup(

:raises NoRemotePathAssignedWithWarmup: If no remote path in repository is assigned.
"""
configure_http_backend(get_requests_session)
if HF_IS_VERSION_0_X_X:
configure_http_backend(get_requests_session)

if not file_in_repo and not dir_in_repo:
raise NoRemotePathAssignedWithWarmup('No remote path in repository assigned.\n'
Expand Down
Loading
Loading