From 7f9a8996097c4836d6eecada4605bd7cbc99fffe Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 5 Feb 2026 21:43:51 +0530 Subject: [PATCH 1/3] Added native hhuggingface native commands and e2e and workflow tests --- .github/workflows/huggingfaceTests.yml | 102 ++++++++ buildtools/cli.go | 86 ++++++ docs/buildtools/huggingface/help.go | 32 +++ go.mod | 2 + go.sum | 4 +- huggingface_test.go | 346 +++++++++++++++++++++++++ main_test.go | 6 + utils/cliutils/commandsflags.go | 21 ++ utils/tests/consts.go | 1 + utils/tests/utils.go | 7 + 10 files changed, 605 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/huggingfaceTests.yml create mode 100644 docs/buildtools/huggingface/help.go create mode 100644 huggingface_test.go diff --git a/.github/workflows/huggingfaceTests.yml b/.github/workflows/huggingfaceTests.yml new file mode 100644 index 000000000..9c45bf963 --- /dev/null +++ b/.github/workflows/huggingfaceTests.yml @@ -0,0 +1,102 @@ +name: HuggingFace Tests +on: + workflow_dispatch: + push: + branches: + - "master" + # Triggers the workflow on PRs to master branch only. + pull_request_target: + types: [labeled] + branches: + - "master" + +# Ensures that only the latest commit is running for each PR at a time. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.ref }} + cancel-in-progress: true +permissions: + id-token: write + contents: read +jobs: + HuggingFace-Tests: + name: HuggingFace tests (${{ matrix.os.name }}) + if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'safe to test') + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + version: 24.04 + - name: windows + version: 2022 + - name: macos + version: 14 + runs-on: ${{ matrix.os.name }}-${{ matrix.os.version }} + steps: + - name: Skip macOS - JGC-413 + if: matrix.os.name == 'macos' + run: | + echo "::warning::JGC-413 - Skip until artifactory bootstrap in osx is fixed" + exit 0 + + - name: Checkout code + if: matrix.os.name != 'macos' + uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + + - name: Setup FastCI + if: matrix.os.name != 'macos' + uses: jfrog-fastci/fastci@v0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + fastci_otel_token: ${{ secrets.FASTCI_TOKEN }} + + - name: Setup Go with cache + if: matrix.os.name != 'macos' + uses: jfrog/.github/actions/install-go-with-cache@main + + - name: Setup Python + if: matrix.os.name != 'macos' + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install HuggingFace CLI + if: matrix.os.name != 'macos' + run: | + pip install huggingface_hub + shell: bash + + - name: Debug macOS Environment and Set Timeout + if: runner.os == 'macOS' + run: | + echo "=== macOS Debug Information ===" + echo "Architecture: $(uname -m)" + echo "macOS Version: $(sw_vers -productVersion)" + echo "macOS Build: $(sw_vers -buildVersion)" + echo "Available memory: $(system_profiler SPHardwareDataType | grep Memory || echo 'Memory info not available')" + echo "Available disk space: $(df -h)" + echo "Go version: $(go version)" + echo "Setting RT_CONNECTION_TIMEOUT_SECONDS to 2400 for macOS" + echo "RT_CONNECTION_TIMEOUT_SECONDS=2400" >> $GITHUB_ENV + + - name: Install local Artifactory + if: matrix.os.name != 'macos' + uses: jfrog/.github/actions/install-local-artifactory@main + with: + RTLIC: ${{ secrets.RTLIC }} + RT_CONNECTION_TIMEOUT_SECONDS: ${{ env.RT_CONNECTION_TIMEOUT_SECONDS || '1200' }} + + - name: Get ID Token and Exchange Token + if: matrix.os.name != 'macos' + shell: bash + run: | + ID_TOKEN=$(curl -sLS -H "User-Agent: actions/oidc-client" -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"') + echo "JFROG_CLI_OIDC_EXCHANGE_TOKEN_ID=${ID_TOKEN}" >> $GITHUB_ENV + + - name: Run HuggingFace tests + if: matrix.os.name != 'macos' + run: go test -v github.com/jfrog/jfrog-cli --timeout 0 --test.huggingface --jfrog.url=http://127.0.0.1:8082 --jfrog.adminToken=${{ env.JFROG_TESTS_LOCAL_ACCESS_TOKEN }} + diff --git a/buildtools/cli.go b/buildtools/cli.go index d9d8c1bd8..84e636d32 100644 --- a/buildtools/cli.go +++ b/buildtools/cli.go @@ -28,6 +28,7 @@ import ( "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/golang" "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/gradle" helmcmd "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/helm" + huggingfaceCommands "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/huggingface" "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/mvn" "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/npm" containerutils "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/ocicontainer" @@ -58,6 +59,7 @@ import ( "github.com/jfrog/jfrog-cli/docs/buildtools/gopublish" gradledoc "github.com/jfrog/jfrog-cli/docs/buildtools/gradle" "github.com/jfrog/jfrog-cli/docs/buildtools/gradleconfig" + "github.com/jfrog/jfrog-cli/docs/buildtools/huggingface" mvndoc "github.com/jfrog/jfrog-cli/docs/buildtools/mvn" "github.com/jfrog/jfrog-cli/docs/buildtools/mvnconfig" "github.com/jfrog/jfrog-cli/docs/buildtools/npmcommand" @@ -475,6 +477,17 @@ func GetCommands() []cli.Command { Category: buildToolsCategory, Action: twineCmd, }, + { + Name: "hugging-face", + Aliases: []string{"hf"}, + Flags: cliutils.GetCommandFlags(cliutils.HuggingFace), + HelpName: corecommon.CreateUsage("hugging-face", huggingface.GetDescription(), huggingface.Usage), + Description: huggingface.GetDescription(), + UsageText: huggingface.GetArguments(), + Hidden: true, + Action: huggingFaceCmd, + Category: buildToolsCategory, + }, }) return decorateWithFlagCapture(cmds) } @@ -1108,6 +1121,79 @@ func loginCmd(c *cli.Context) error { return nil } +func huggingFaceCmd(c *cli.Context) error { + if show, err := cliutils.ShowCmdHelpIfNeeded(c, c.Args()); show || err != nil { + return err + } + if c.NArg() < 1 { + return cliutils.WrongNumberOfArgumentsHandler(c) + } + args := cliutils.ExtractCommand(c) + cmdName, hfArgs := getCommandName(args) + switch cmdName { + case "u", "upload": + return huggingFaceUploadCmd(c, hfArgs) + case "d", "download": + return huggingFaceDownloadCmd(c, hfArgs) + default: + return errors.New("Wrong command: " + cmdName) + } +} + +func huggingFaceUploadCmd(c *cli.Context, hfArgs []string) error { + // Upload requires folderPath and repoID + if len(hfArgs) < 2 { + return cliutils.PrintHelpAndReturnError("Folder path and repository ID are required.", c) + } + folderPath := hfArgs[0] + if folderPath == "" { + return cliutils.PrintHelpAndReturnError("Folder path cannot be empty.", c) + } + repoID := hfArgs[1] + if repoID == "" { + return cliutils.PrintHelpAndReturnError("Repository ID cannot be empty.", c) + } + revision := "" + if c.String("revision") != "" { + revision = c.String("revision") + } + repoType := c.String("repo-type") + if repoType == "" { + repoType = "model" + } + huggingFaceUploadCmd := huggingfaceCommands.NewHuggingFaceUpload().SetFolderPath(folderPath).SetRepoId(repoID).SetRepoType(repoType).SetRevision(revision) + return commands.Exec(huggingFaceUploadCmd) +} + +func huggingFaceDownloadCmd(c *cli.Context, hfArgs []string) error { + // Download requires repoID + if len(hfArgs) < 1 { + return cliutils.PrintHelpAndReturnError("Model/Dataset name is required.", c) + } + repoID := hfArgs[0] + if repoID == "" { + return cliutils.PrintHelpAndReturnError("Model/Dataset name cannot be empty.", c) + } + revision := "" + if c.String("revision") != "" { + revision = c.String("revision") + } + etagTimeout := 86400 + if c.String("etag-timeout") != "" { + var err error + etagTimeout, err = strconv.Atoi(c.String("etag-timeout")) + if err != nil { + return errorutils.CheckErrorf("invalid etag-timeout value: %s", c.String("etag-timeout")) + } + } + repoType := c.String("repo-type") + if repoType == "" { + repoType = "model" + } + huggingFaceDownloadCmd := huggingfaceCommands.NewHuggingFaceDownload().SetRepoId(repoID).SetRepoType(repoType).SetRevision(revision).SetEtagTimeout(etagTimeout) + return commands.Exec(huggingFaceDownloadCmd) +} + func dockerScanCmd(c *cli.Context, imageTag string) error { if show, err := cliutils.ShowGenericCmdHelpIfNeeded(c, c.Args(), securityCLI.DockerScanCmdHiddenName); show || err != nil { return err diff --git a/docs/buildtools/huggingface/help.go b/docs/buildtools/huggingface/help.go new file mode 100644 index 000000000..de94f2095 --- /dev/null +++ b/docs/buildtools/huggingface/help.go @@ -0,0 +1,32 @@ +package huggingface + +var Usage = []string{"hf d ", + "hf u "} + +func GetDescription() string { + return `Download or upload models/datasets from/to HuggingFace Hub.` +} + +func GetArguments() string { + return ` d + Download a model/dataset from HuggingFace Hub. + model-name + The HuggingFace model repository ID (e.g., 'bert-base-uncased' or 'username/model-name'). + + u + Upload a model or dataset folder to HuggingFace Hub. + folder-path + Path to the folder to upload. + repo-id + The HuggingFace repository ID (e.g., 'username/model-name' or 'username/dataset-name'). + + Command options: + --revision + [Optional] The revision (commit hash, branch name, or tag) to download/upload. If not specified, uses the default branch. + + --repo-type + [Optional] The repository type. Can be 'model' or 'dataset'. Default: 'model'. + + --etag-timeout + [Optional] [Download only] Timeout in seconds for ETag validation. Default: 86400 (24 hours).` +} diff --git a/go.mod b/go.mod index 826e04e5c..dea6f6515 100644 --- a/go.mod +++ b/go.mod @@ -257,6 +257,8 @@ replace github.com/gfleury/go-bitbucket-v1 => github.com/gfleury/go-bitbucket-v1 replace github.com/ktrysmt/go-bitbucket => github.com/ktrysmt/go-bitbucket v0.9.80 +replace github.com/jfrog/jfrog-cli-artifactory => github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d + //replace github.com/jfrog/jfrog-cli-core/v2 => ../jfrog-cli-core //replace github.com/jfrog/build-info-go => github.com/fluxxBot/build-info-go v1.10.10-0.20260105070825-d3f36f619ba5 diff --git a/go.sum b/go.sum index 4191e2e77..7836bf780 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,6 @@ github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYL github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= github.com/jfrog/jfrog-cli-application v1.0.2-0.20260202095705-3657239cde88 h1:KP6SmrduuGMMsMfHhBATv5I3W1iTtjBojtqEkPHBWk4= github.com/jfrog/jfrog-cli-application v1.0.2-0.20260202095705-3657239cde88/go.mod h1:xum2HquWO5uExa/A7MQs3TgJJVEeoqTR+6Z4mfBr1Xw= -github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260204122910-dcfb1d59b122 h1:fapP2b7hTJbrYuy8P/WSo84nwzYK5FnRaxNonD2NOvc= -github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260204122910-dcfb1d59b122/go.mod h1:1GKFtQs/5/3HRp1JyMHMQVT9lxzXR+YecO6dWdRThNs= github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20260202114755-cec5fd702f50 h1:u3NvevCPC5ygeOMv39XSlpIOCt7PhLtFWR2XZ0QkKaU= github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20260202114755-cec5fd702f50/go.mod h1:+Hnaikp/xCSPD/q7txxRy4Zc0wzjW/usrCSf+6uONSQ= github.com/jfrog/jfrog-cli-evidence v0.8.3-0.20260202100913-d9ee9476845a h1:lTOAhUjKcOmM/0Kbj4V+I/VHPlW7YNAhIEVpGnCM5mI= @@ -522,6 +520,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= +github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d h1:v7iPT3Udy+wZoWoqR0kkV7xB+Hby29l0T0YDsnqpY3c= +github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d/go.mod h1:1GKFtQs/5/3HRp1JyMHMQVT9lxzXR+YecO6dWdRThNs= github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= diff --git a/huggingface_test.go b/huggingface_test.go new file mode 100644 index 000000000..2e2d9517e --- /dev/null +++ b/huggingface_test.go @@ -0,0 +1,346 @@ +package main + +// HuggingFace Integration Tests +// Run with: go test -v -test.huggingface -jfrog.url=http://localhost:8081/ -jfrog.user=admin -jfrog.password=password + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/jfrog/jfrog-cli-core/v2/utils/config" + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + coreTests "github.com/jfrog/jfrog-cli-core/v2/utils/tests" + "github.com/jfrog/jfrog-cli/utils/tests" + clientTestUtils "github.com/jfrog/jfrog-client-go/utils/tests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func initHuggingFaceTest(t *testing.T) { + if !*tests.TestHuggingFace { + t.Skip("Skipping HuggingFace test. To run HuggingFace test add the '-test.huggingface=true' option.") + } + + if artifactoryCli == nil { + initArtifactoryCli() + } + + // Set up home directory configuration so GetDefaultServerConf() can find the server + createJfrogHomeConfig(t, true) + + // Initialize serverDetails for HuggingFace tests + serverDetails = &config.ServerDetails{ + Url: *tests.JfrogUrl, + ArtifactoryUrl: *tests.JfrogUrl + tests.ArtifactoryEndpoint, + SshKeyPath: *tests.JfrogSshKeyPath, + SshPassphrase: *tests.JfrogSshPassphrase, + } + if *tests.JfrogAccessToken != "" { + serverDetails.AccessToken = *tests.JfrogAccessToken + } else { + serverDetails.User = *tests.JfrogUser + serverDetails.Password = *tests.JfrogPassword + } +} + +func cleanHuggingFaceTest(t *testing.T) { + clientTestUtils.UnSetEnvAndAssert(t, coreutils.HomeDir) + tests.CleanFileSystem() +} + +// TestHuggingFaceDownload tests the HuggingFace download command +func TestHuggingFaceDownload(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace download test") + } + + // Test download with a small test model + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test basic download command structure (dry run style test) + // Using a well-known small model for testing + args := []string{ + "hf", "d", "gpt2", + "--repo-type=model", + } + + // Execute and check for proper command handling + // Note: This test verifies command parsing and execution flow + // Actual download depends on HuggingFace Hub availability + err := jfrogCli.Exec(args...) + + // The command should either succeed or fail gracefully with a network/auth error + // We're testing the CLI integration, not the actual HuggingFace Hub connectivity + if err != nil { + // Check if error is due to missing HuggingFace token (expected in CI) + t.Logf("HuggingFace download command returned: %v (this may be expected in CI without HF token)", err) + } +} + +// TestHuggingFaceDownloadWithRevision tests the HuggingFace download command with revision parameter +func TestHuggingFaceDownloadWithRevision(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace download test") + } + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test download with revision parameter + args := []string{ + "hf", "d", "gpt2", + "--repo-type=model", + "--revision=main", + } + + err := jfrogCli.Exec(args...) + if err != nil { + t.Logf("HuggingFace download with revision command returned: %v (this may be expected in CI without HF token)", err) + } +} + +// TestHuggingFaceDownloadDataset tests the HuggingFace download command for datasets +func TestHuggingFaceDownloadDataset(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace download test") + } + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test download dataset + args := []string{ + "hf", "d", "squad", + "--repo-type=dataset", + } + + err := jfrogCli.Exec(args...) + if err != nil { + t.Logf("HuggingFace download dataset command returned: %v (this may be expected in CI without HF token)", err) + } +} + +// TestHuggingFaceDownloadWithEtagTimeout tests the HuggingFace download command with etag-timeout +func TestHuggingFaceDownloadWithEtagTimeout(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace download test") + } + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test download with etag-timeout parameter + args := []string{ + "hf", "d", "gpt2", + "--repo-type=model", + "--etag-timeout=3600", + } + + err := jfrogCli.Exec(args...) + if err != nil { + t.Logf("HuggingFace download with etag-timeout command returned: %v (this may be expected in CI without HF token)", err) + } +} + +// TestHuggingFaceUpload tests the HuggingFace upload command +func TestHuggingFaceUpload(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace upload test") + } + + // Create a temporary directory with test files to upload + tempDir, err := os.MkdirTemp("", "hf-upload-test-*") + require.NoError(t, err) + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("Warning: Failed to remove temp directory %s: %v", tempDir, err) + } + }() + + // Create a test file in the temp directory + testFile := filepath.Join(tempDir, "test_model.txt") + err = os.WriteFile(testFile, []byte("test model content"), 0644) + require.NoError(t, err) + + // Create a model config file + configFile := filepath.Join(tempDir, "config.json") + err = os.WriteFile(configFile, []byte(`{"model_type": "test"}`), 0644) + require.NoError(t, err) + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test upload command structure + // Note: This will require HuggingFace credentials to actually upload + args := []string{ + "hf", "u", tempDir, "test-org/test-model", + "--repo-type=model", + } + + err = jfrogCli.Exec(args...) + if err != nil { + // Expected to fail without proper HuggingFace credentials + t.Logf("HuggingFace upload command returned: %v (this is expected without HF credentials)", err) + } +} + +// TestHuggingFaceUploadWithRevision tests the HuggingFace upload command with revision parameter +func TestHuggingFaceUploadWithRevision(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace upload test") + } + + // Create a temporary directory with test files to upload + tempDir, err := os.MkdirTemp("", "hf-upload-revision-test-*") + require.NoError(t, err) + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("Warning: Failed to remove temp directory %s: %v", tempDir, err) + } + }() + + // Create a test file in the temp directory + testFile := filepath.Join(tempDir, "test_model.txt") + err = os.WriteFile(testFile, []byte("test model content for revision test"), 0644) + require.NoError(t, err) + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test upload with revision parameter + args := []string{ + "hf", "u", tempDir, "test-org/test-model", + "--repo-type=model", + "--revision=test-branch", + } + + err = jfrogCli.Exec(args...) + if err != nil { + t.Logf("HuggingFace upload with revision command returned: %v (this is expected without HF credentials)", err) + } +} + +// TestHuggingFaceUploadDataset tests the HuggingFace upload command for datasets +func TestHuggingFaceUploadDataset(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + // Check if huggingface-cli is available + if _, err := exec.LookPath("huggingface-cli"); err != nil { + t.Skip("huggingface-cli not found in PATH, skipping HuggingFace upload test") + } + + // Create a temporary directory with test dataset files + tempDir, err := os.MkdirTemp("", "hf-upload-dataset-test-*") + require.NoError(t, err) + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("Warning: Failed to remove temp directory %s: %v", tempDir, err) + } + }() + + // Create test dataset files + trainFile := filepath.Join(tempDir, "train.json") + err = os.WriteFile(trainFile, []byte(`[{"text": "sample training data"}]`), 0644) + require.NoError(t, err) + + testFileData := filepath.Join(tempDir, "test.json") + err = os.WriteFile(testFileData, []byte(`[{"text": "sample test data"}]`), 0644) + require.NoError(t, err) + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test upload dataset + args := []string{ + "hf", "u", tempDir, "test-org/test-dataset", + "--repo-type=dataset", + } + + err = jfrogCli.Exec(args...) + if err != nil { + t.Logf("HuggingFace upload dataset command returned: %v (this is expected without HF credentials)", err) + } +} + +// TestHuggingFaceCommandValidation tests that the HuggingFace command properly validates arguments +func TestHuggingFaceCommandValidation(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test download without model name should fail + args := []string{ + "hf", "d", + } + err := jfrogCli.Exec(args...) + assert.Error(t, err, "Download without model name should fail") + + // Test upload without folder path and repo-id should fail + args = []string{ + "hf", "u", + } + err = jfrogCli.Exec(args...) + assert.Error(t, err, "Upload without folder path and repo-id should fail") + + // Test upload with only folder path should fail + args = []string{ + "hf", "u", "/tmp/test-folder", + } + err = jfrogCli.Exec(args...) + assert.Error(t, err, "Upload with only folder path should fail") + + // Test invalid subcommand should fail + args = []string{ + "hf", "invalid", + } + err = jfrogCli.Exec(args...) + assert.Error(t, err, "Invalid subcommand should fail") +} + +// TestHuggingFaceHelp tests that the HuggingFace help is displayed correctly +func TestHuggingFaceHelp(t *testing.T) { + initHuggingFaceTest(t) + defer cleanHuggingFaceTest(t) + + jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") + + // Test help flag + args := []string{ + "hf", "--help", + } + err := jfrogCli.Exec(args...) + assert.NoError(t, err, "Help command should not return error") +} + +// InitHuggingFaceTests initializes HuggingFace tests +func InitHuggingFaceTests() { + initArtifactoryCli() +} + +// CleanHuggingFaceTests cleans up after HuggingFace tests +func CleanHuggingFaceTests() { + // Cleanup is handled per-test +} + diff --git a/main_test.go b/main_test.go index 5b19b2689..430682217 100644 --- a/main_test.go +++ b/main_test.go @@ -94,6 +94,9 @@ func setupIntegrationTests() { if *tests.TestHelm { InitHelmTests() } + if *tests.TestHuggingFace { + InitHuggingFaceTests() + } } func tearDownIntegrationTests() { @@ -121,6 +124,9 @@ func tearDownIntegrationTests() { if *tests.TestHelm { CleanHelmTests() } + if *tests.TestHuggingFace { + CleanHuggingFaceTests() + } } func InitBuildToolsTests() { diff --git a/utils/cliutils/commandsflags.go b/utils/cliutils/commandsflags.go index 8d7768a0d..b7a5e85f9 100644 --- a/utils/cliutils/commandsflags.go +++ b/utils/cliutils/commandsflags.go @@ -78,6 +78,7 @@ const ( PoetryConfig = "poetry-config" Poetry = "poetry" Helm = "helm" + HuggingFace = "hugging-face" RubyConfig = "ruby-config" Conan = "conan" Ping = "ping" @@ -593,6 +594,11 @@ const ( setupRepo = repo PromotionType = "promotion-type" Draft = "draft" + + // HuggingFace flags + Revision = "revision" + RepoType = "repo-type" + EtagTimeout = "etag-timeout" ) var flagsMap = map[string]cli.Flag{ @@ -1749,6 +1755,18 @@ var flagsMap = map[string]cli.Flag{ Name: validateSha, Usage: "[Default: false] Set to true to enable SHA validation during Docker push.` `", }, + Revision: cli.StringFlag{ + Name: Revision, + Usage: "[Default: main]The specific revision, branch, or tag to download. Defaults to main branch if not specified.` `", + }, + EtagTimeout: cli.StringFlag{ + Name: EtagTimeout, + Usage: "Timeout in seconds for ETag validation. Defaults to 86400 (24 hours).` `", + }, + RepoType: cli.StringFlag{ + Name: RepoType, + Usage: "[Default: model] Type of repository. Can be 'model', 'dataset'.` `", + }, } var commandFlags = map[string][]string{ @@ -1981,6 +1999,9 @@ var commandFlags = map[string][]string{ Helm: { BuildName, BuildNumber, module, Project, serverId, username, password, }, + HuggingFace: { + Revision, RepoType, EtagTimeout, + }, RubyConfig: { global, serverIdResolve, serverIdDeploy, repoResolve, repoDeploy, }, diff --git a/utils/tests/consts.go b/utils/tests/consts.go index ea40bab8a..ad7b8e1d1 100644 --- a/utils/tests/consts.go +++ b/utils/tests/consts.go @@ -248,6 +248,7 @@ var ( PoetryBuildName = "cli-poetry-build" ConanBuildName = "cli-conan-build" HelmBuildName = "cli-helm-build" + HuggingFaceBuildName = "cli-huggingface-build" RtBuildName1 = "cli-rt-build1" RtBuildName2 = "cli-rt-build2" RtBuildNameWithSpecialChars = "cli-rt-a$+~&^a#-build3" diff --git a/utils/tests/utils.go b/utils/tests/utils.go index 940373238..a398b626a 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -70,6 +70,7 @@ var ( TestPoetry *bool TestConan *bool TestHelm *bool + TestHuggingFace *bool TestPlugins *bool TestXray *bool TestAccess *bool @@ -109,6 +110,7 @@ func init() { TestPoetry = flag.Bool("test.poetry", false, "Test Poetry") TestConan = flag.Bool("test.conan", false, "Test Conan") TestHelm = flag.Bool("test.helm", false, "Test Helm") + TestHuggingFace = flag.Bool("test.huggingface", false, "Test HuggingFace") TestPlugins = flag.Bool("test.plugins", false, "Test Plugins") TestXray = flag.Bool("test.xray", false, "Test Xray") TestAccess = flag.Bool("test.access", false, "Test Access") @@ -353,12 +355,14 @@ func GetNonVirtualRepositories() map[*string]string { TestPoetry: {&PoetryLocalRepo, &PoetryRemoteRepo}, TestConan: {&ConanLocalRepo, &ConanRemoteRepo}, TestHelm: {&HelmLocalRepo}, + TestHuggingFace: {}, TestPlugins: {&RtRepo1}, TestXray: {&NpmRemoteRepo, &NugetRemoteRepo, &YarnRemoteRepo, &GradleRemoteRepo, &MvnRemoteRepo, &GoRepo, &GoRemoteRepo, &PypiRemoteRepo}, TestAccess: {&RtRepo1}, TestTransfer: {&RtRepo1, &RtRepo2, &MvnRepo1, &MvnRemoteRepo, &DockerRemoteRepo}, TestLifecycle: {&RtDevRepo, &RtProdRepo1, &RtProdRepo2}, TestHelm: {&RtRepo1}, + TestHuggingFace: {}, } return getNeededRepositories(nonVirtualReposMap) } @@ -381,9 +385,11 @@ func GetVirtualRepositories() map[*string]string { TestPoetry: {&PoetryVirtualRepo}, TestConan: {&ConanVirtualRepo}, TestHelm: {}, + TestHuggingFace: {}, TestPlugins: {}, TestXray: {&GoVirtualRepo}, TestAccess: {}, + TestHuggingFace: {}, TestHelm: {}, } return getNeededRepositories(virtualReposMap) @@ -420,6 +426,7 @@ func GetBuildNames() []string { TestPoetry: {&PoetryBuildName}, TestConan: {&ConanBuildName}, TestHelm: {&HelmBuildName}, + TestHuggingFace: {&HuggingFaceBuildName}, TestPlugins: {}, TestXray: {}, TestAccess: {}, From 96bdd846c8f09f6568d01671b13dba9dffa08b63 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 10 Feb 2026 11:36:13 +0530 Subject: [PATCH 2/3] Updated jfrog-cli-artifactory version in go.mod and go.sum --- go.mod | 4 ++-- go.sum | 6 ++---- huggingface_test.go | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 1e7f6e257..e82ec1ca9 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/jfrog/build-info-go v1.13.1-0.20260130140656-2d0d5593fccf github.com/jfrog/gofrog v1.7.6 github.com/jfrog/jfrog-cli-application v1.0.2-0.20260202095705-3657239cde88 - github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260206111107-7a05b41040dd + github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260210060138-bbe22834d469 github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20260202114755-cec5fd702f50 github.com/jfrog/jfrog-cli-evidence v0.8.3-0.20260202100913-d9ee9476845a github.com/jfrog/jfrog-cli-platform-services v1.10.1-0.20251205121610-171eb9b0000e @@ -257,7 +257,7 @@ replace github.com/gfleury/go-bitbucket-v1 => github.com/gfleury/go-bitbucket-v1 replace github.com/ktrysmt/go-bitbucket => github.com/ktrysmt/go-bitbucket v0.9.80 -replace github.com/jfrog/jfrog-cli-artifactory => github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d +//replace github.com/jfrog/jfrog-cli-artifactory => github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d //replace github.com/jfrog/jfrog-cli-core/v2 => ../jfrog-cli-core diff --git a/go.sum b/go.sum index bd13156d5..fedcbc5f8 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYL github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= github.com/jfrog/jfrog-cli-application v1.0.2-0.20260202095705-3657239cde88 h1:KP6SmrduuGMMsMfHhBATv5I3W1iTtjBojtqEkPHBWk4= github.com/jfrog/jfrog-cli-application v1.0.2-0.20260202095705-3657239cde88/go.mod h1:xum2HquWO5uExa/A7MQs3TgJJVEeoqTR+6Z4mfBr1Xw= -github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260206111107-7a05b41040dd h1:ZKRwlPEkSmul7fBKat7qBVU3tSxd6xg2GbxQixK4COg= -github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260206111107-7a05b41040dd/go.mod h1:1GKFtQs/5/3HRp1JyMHMQVT9lxzXR+YecO6dWdRThNs= +github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260210060138-bbe22834d469 h1:uCC03HffS+KPfd9rmW61jAmv7fi8FyJwUrw4FfioIYM= +github.com/jfrog/jfrog-cli-artifactory v0.8.1-0.20260210060138-bbe22834d469/go.mod h1:1GKFtQs/5/3HRp1JyMHMQVT9lxzXR+YecO6dWdRThNs= github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20260202114755-cec5fd702f50 h1:u3NvevCPC5ygeOMv39XSlpIOCt7PhLtFWR2XZ0QkKaU= github.com/jfrog/jfrog-cli-core/v2 v2.60.1-0.20260202114755-cec5fd702f50/go.mod h1:+Hnaikp/xCSPD/q7txxRy4Zc0wzjW/usrCSf+6uONSQ= github.com/jfrog/jfrog-cli-evidence v0.8.3-0.20260202100913-d9ee9476845a h1:lTOAhUjKcOmM/0Kbj4V+I/VHPlW7YNAhIEVpGnCM5mI= @@ -522,8 +522,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= -github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d h1:v7iPT3Udy+wZoWoqR0kkV7xB+Hby29l0T0YDsnqpY3c= -github.com/naveenku-jfrog/jfrog-cli-artifactory v0.0.0-20260205125834-9726cdd29d6d/go.mod h1:1GKFtQs/5/3HRp1JyMHMQVT9lxzXR+YecO6dWdRThNs= github.com/nwaples/rardecode/v2 v2.2.2 h1:/5oL8dzYivRM/tqX9VcTSWfbpwcbwKG1QtSJr3b3KcU= github.com/nwaples/rardecode/v2 v2.2.2/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= diff --git a/huggingface_test.go b/huggingface_test.go index 2e2d9517e..6fae75d38 100644 --- a/huggingface_test.go +++ b/huggingface_test.go @@ -343,4 +343,3 @@ func InitHuggingFaceTests() { func CleanHuggingFaceTests() { // Cleanup is handled per-test } - From 4451840ff48e2a4495262c18b6ef3e0209b4f47f Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 13 Feb 2026 14:22:02 +0530 Subject: [PATCH 3/3] Addressed comments --- buildtools/cli.go | 5 +++-- docs/buildtools/huggingface/help.go | 4 ++-- huggingface_test.go | 34 +++++++++++++++++------------ utils/cliutils/commandsflags.go | 2 +- utils/tests/utils.go | 2 -- 5 files changed, 26 insertions(+), 21 deletions(-) diff --git a/buildtools/cli.go b/buildtools/cli.go index f5b3a17fa..c0aa5006e 100644 --- a/buildtools/cli.go +++ b/buildtools/cli.go @@ -1143,7 +1143,7 @@ func huggingFaceCmd(c *cli.Context) error { case "d", "download": return huggingFaceDownloadCmd(c, hfArgs) default: - return errors.New("Wrong command: " + cmdName) + return errorutils.CheckErrorf("unknown HuggingFace command: '%s'. Valid commands are: upload (u), download (d)", cmdName) } } @@ -1177,6 +1177,7 @@ func huggingFaceDownloadCmd(c *cli.Context, hfArgs []string) error { if len(hfArgs) < 1 { return cliutils.PrintHelpAndReturnError("Model/Dataset name is required.", c) } + const eTagTimeout = 86400 repoID := hfArgs[0] if repoID == "" { return cliutils.PrintHelpAndReturnError("Model/Dataset name cannot be empty.", c) @@ -1185,7 +1186,7 @@ func huggingFaceDownloadCmd(c *cli.Context, hfArgs []string) error { if c.String("revision") != "" { revision = c.String("revision") } - etagTimeout := 86400 + etagTimeout := eTagTimeout if c.String("etag-timeout") != "" { var err error etagTimeout, err = strconv.Atoi(c.String("etag-timeout")) diff --git a/docs/buildtools/huggingface/help.go b/docs/buildtools/huggingface/help.go index de94f2095..cf4952d00 100644 --- a/docs/buildtools/huggingface/help.go +++ b/docs/buildtools/huggingface/help.go @@ -22,11 +22,11 @@ func GetArguments() string { Command options: --revision - [Optional] The revision (commit hash, branch name, or tag) to download/upload. If not specified, uses the default branch. + [Optional] The revision (commit hash, branch name, or tag) to download/upload. Defaults to main branch if not specified. --repo-type [Optional] The repository type. Can be 'model' or 'dataset'. Default: 'model'. --etag-timeout - [Optional] [Download only] Timeout in seconds for ETag validation. Default: 86400 (24 hours).` + [Optional] [Download only] Timeout in seconds for ETag validation. Default: 86400 seconds (24 hours).` } diff --git a/huggingface_test.go b/huggingface_test.go index 6fae75d38..9f83d7d68 100644 --- a/huggingface_test.go +++ b/huggingface_test.go @@ -169,22 +169,24 @@ func TestHuggingFaceUpload(t *testing.T) { // Create a temporary directory with test files to upload tempDir, err := os.MkdirTemp("", "hf-upload-test-*") - require.NoError(t, err) - defer func() { + if err != nil { + require.NoError(t, err, "Failed to create temp directory") + } + t.Cleanup(func() { if err := os.RemoveAll(tempDir); err != nil { t.Logf("Warning: Failed to remove temp directory %s: %v", tempDir, err) } - }() + }) // Create a test file in the temp directory testFile := filepath.Join(tempDir, "test_model.txt") err = os.WriteFile(testFile, []byte("test model content"), 0644) - require.NoError(t, err) + require.NoError(t, err, "Failed to create test model file") // Create a model config file configFile := filepath.Join(tempDir, "config.json") err = os.WriteFile(configFile, []byte(`{"model_type": "test"}`), 0644) - require.NoError(t, err) + require.NoError(t, err, "Failed to create config file") jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") @@ -214,17 +216,19 @@ func TestHuggingFaceUploadWithRevision(t *testing.T) { // Create a temporary directory with test files to upload tempDir, err := os.MkdirTemp("", "hf-upload-revision-test-*") - require.NoError(t, err) - defer func() { + if err != nil { + require.NoError(t, err, "Failed to create temp directory") + } + t.Cleanup(func() { if err := os.RemoveAll(tempDir); err != nil { t.Logf("Warning: Failed to remove temp directory %s: %v", tempDir, err) } - }() + }) // Create a test file in the temp directory testFile := filepath.Join(tempDir, "test_model.txt") err = os.WriteFile(testFile, []byte("test model content for revision test"), 0644) - require.NoError(t, err) + require.NoError(t, err, "Failed to create test model file") jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") @@ -253,21 +257,23 @@ func TestHuggingFaceUploadDataset(t *testing.T) { // Create a temporary directory with test dataset files tempDir, err := os.MkdirTemp("", "hf-upload-dataset-test-*") - require.NoError(t, err) - defer func() { + if err != nil { + require.NoError(t, err, "Failed to create temp directory") + } + t.Cleanup(func() { if err := os.RemoveAll(tempDir); err != nil { t.Logf("Warning: Failed to remove temp directory %s: %v", tempDir, err) } - }() + }) // Create test dataset files trainFile := filepath.Join(tempDir, "train.json") err = os.WriteFile(trainFile, []byte(`[{"text": "sample training data"}]`), 0644) - require.NoError(t, err) + require.NoError(t, err, "Failed to create train file") testFileData := filepath.Join(tempDir, "test.json") err = os.WriteFile(testFileData, []byte(`[{"text": "sample test data"}]`), 0644) - require.NoError(t, err) + require.NoError(t, err, "Failed to create test file") jfrogCli := coreTests.NewJfrogCli(execMain, "jfrog", "") diff --git a/utils/cliutils/commandsflags.go b/utils/cliutils/commandsflags.go index b7a5e85f9..010a77a44 100644 --- a/utils/cliutils/commandsflags.go +++ b/utils/cliutils/commandsflags.go @@ -1757,7 +1757,7 @@ var flagsMap = map[string]cli.Flag{ }, Revision: cli.StringFlag{ Name: Revision, - Usage: "[Default: main]The specific revision, branch, or tag to download. Defaults to main branch if not specified.` `", + Usage: "[Default: main] The specific revision, branch, or tag to download. Defaults to main branch if not specified.` `", }, EtagTimeout: cli.StringFlag{ Name: EtagTimeout, diff --git a/utils/tests/utils.go b/utils/tests/utils.go index a398b626a..f74c1ecaf 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -362,7 +362,6 @@ func GetNonVirtualRepositories() map[*string]string { TestTransfer: {&RtRepo1, &RtRepo2, &MvnRepo1, &MvnRemoteRepo, &DockerRemoteRepo}, TestLifecycle: {&RtDevRepo, &RtProdRepo1, &RtProdRepo2}, TestHelm: {&RtRepo1}, - TestHuggingFace: {}, } return getNeededRepositories(nonVirtualReposMap) } @@ -389,7 +388,6 @@ func GetVirtualRepositories() map[*string]string { TestPlugins: {}, TestXray: {&GoVirtualRepo}, TestAccess: {}, - TestHuggingFace: {}, TestHelm: {}, } return getNeededRepositories(virtualReposMap)