diff --git a/.github/workflows/cron-licenses.yml b/.github/workflows/cron-licenses.yml index 12f52289b6..a8be1ffa59 100644 --- a/.github/workflows/cron-licenses.yml +++ b/.github/workflows/cron-licenses.yml @@ -9,8 +9,10 @@ jobs: cron-licenses: runs-on: ubuntu-latest if: github.repository == 'go-gitea/gitea' + permissions: + contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/cron-translations.yml b/.github/workflows/cron-translations.yml index ae2238ad2d..3a012e9876 100644 --- a/.github/workflows/cron-translations.yml +++ b/.github/workflows/cron-translations.yml @@ -9,8 +9,10 @@ jobs: crowdin-pull: runs-on: ubuntu-latest if: github.repository == 'go-gitea/gitea' + permissions: + contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: crowdin/github-action@v1 with: upload_sources: true diff --git a/.github/workflows/files-changed.yml b/.github/workflows/files-changed.yml index b21341a277..d18ee6e998 100644 --- a/.github/workflows/files-changed.yml +++ b/.github/workflows/files-changed.yml @@ -24,6 +24,8 @@ jobs: detect: runs-on: ubuntu-latest timeout-minutes: 3 + permissions: + contents: read outputs: backend: ${{ steps.changes.outputs.backend }} frontend: ${{ steps.changes.outputs.frontend }} @@ -34,7 +36,7 @@ jobs: swagger: ${{ steps.changes.outputs.swagger }} yaml: ${{ steps.changes.outputs.yaml }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dorny/paths-filter@v3 id: changes with: diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml index f73772e934..c146b439e0 100644 --- a/.github/workflows/pull-compliance.yml +++ b/.github/workflows/pull-compliance.yml @@ -10,13 +10,17 @@ concurrency: jobs: files-changed: uses: ./.github/workflows/files-changed.yml + permissions: + contents: read lint-backend: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -30,8 +34,10 @@ jobs: if: needs.files-changed.outputs.templates == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@v6 - run: uv python install 3.12 - uses: pnpm/action-setup@v4 @@ -46,8 +52,10 @@ jobs: if: needs.files-changed.outputs.yaml == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: astral-sh/setup-uv@v6 - run: uv python install 3.12 - run: make deps-py @@ -57,8 +65,10 @@ jobs: if: needs.files-changed.outputs.swagger == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v5 with: @@ -70,8 +80,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.frontend == 'true' || needs.files-changed.outputs.actions == 'true' || needs.files-changed.outputs.docs == 'true' || needs.files-changed.outputs.templates == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -82,8 +94,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -99,8 +113,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -114,8 +130,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -127,8 +145,10 @@ jobs: if: needs.files-changed.outputs.frontend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v5 with: @@ -143,8 +163,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -175,8 +197,10 @@ jobs: if: needs.files-changed.outputs.docs == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pnpm/action-setup@v4 - uses: actions/setup-node@v5 with: @@ -188,8 +212,10 @@ jobs: if: needs.files-changed.outputs.actions == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 21ec76b48e..66f48d5af8 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -10,11 +10,15 @@ concurrency: jobs: files-changed: uses: ./.github/workflows/files-changed.yml + permissions: + contents: read test-pgsql: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: pgsql: image: postgres:14 @@ -38,7 +42,7 @@ jobs: ports: - "9000:9000" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -65,8 +69,10 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -90,6 +96,8 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: elasticsearch: image: elasticsearch:7.5.0 @@ -124,7 +132,7 @@ jobs: ports: - 10000:10000 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -152,6 +160,8 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: mysql: # the bitnami mysql image has more options than the official one, it's easier to customize @@ -177,7 +187,7 @@ jobs: - "587:587" - "993:993" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -203,6 +213,8 @@ jobs: if: needs.files-changed.outputs.backend == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read services: mssql: image: mcr.microsoft.com/mssql/server:2019-latest @@ -217,7 +229,7 @@ jobs: ports: - 10000:10000 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version-file: go.mod diff --git a/.github/workflows/pull-docker-dryrun.yml b/.github/workflows/pull-docker-dryrun.yml index 9c9dd2ffe6..1cd1ba31dd 100644 --- a/.github/workflows/pull-docker-dryrun.yml +++ b/.github/workflows/pull-docker-dryrun.yml @@ -10,13 +10,17 @@ concurrency: jobs: files-changed: uses: ./.github/workflows/files-changed.yml + permissions: + contents: read container: if: needs.files-changed.outputs.docker == 'true' || needs.files-changed.outputs.actions == 'true' needs: files-changed runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: docker/setup-buildx-action@v3 - name: Build regular container image uses: docker/build-push-action@v5 diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index ada4c18d33..c8ce0aa787 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -11,8 +11,10 @@ concurrency: jobs: nightly-binary: runs-on: namespace-profile-gitea-release-binary + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force @@ -56,12 +58,14 @@ jobs: - name: upload binaries to s3 run: | aws s3 sync dist/release s3://${{ secrets.AWS_S3_BUCKET }}/gitea/${{ steps.clean_name.outputs.branch }} --no-progress + nightly-container: runs-on: namespace-profile-gitea-release-docker permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force diff --git a/.github/workflows/release-tag-rc.yml b/.github/workflows/release-tag-rc.yml index 35558933e0..ef36e55a94 100644 --- a/.github/workflows/release-tag-rc.yml +++ b/.github/workflows/release-tag-rc.yml @@ -12,8 +12,10 @@ concurrency: jobs: binary: runs-on: namespace-profile-gitea-release-binary + permissions: + contents: read steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force @@ -66,12 +68,14 @@ jobs: gh release create ${{ github.ref_name }} --title ${{ github.ref_name }} --draft --notes-from-tag dist/release/* env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + container: runs-on: namespace-profile-gitea-release-docker permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force diff --git a/.github/workflows/release-tag-version.yml b/.github/workflows/release-tag-version.yml index 56426d3bc3..a3838de3c0 100644 --- a/.github/workflows/release-tag-version.yml +++ b/.github/workflows/release-tag-version.yml @@ -15,9 +15,10 @@ jobs: binary: runs-on: namespace-profile-gitea-release-binary permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force @@ -70,12 +71,14 @@ jobs: gh release create ${{ github.ref_name }} --title ${{ github.ref_name }} --notes-from-tag dist/release/* env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + container: runs-on: namespace-profile-gitea-release-docker permissions: + contents: read packages: write # to publish to ghcr.io steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # fetch all commits instead of only the last as some branches are long lived and could have many between versions # fetch all tags to ensure that "git describe" reports expected Gitea version, eg. v1.21.0-dev-1-g1234567 - run: git fetch --unshallow --quiet --tags --force diff --git a/models/user/search.go b/models/user/search.go index db4b07f64a..36d1d3913b 100644 --- a/models/user/search.go +++ b/models/user/search.go @@ -18,6 +18,23 @@ import ( "xorm.io/xorm" ) +// AdminUserOrderByMap represents all possible admin user search orders +// This should only be used for admin API endpoints as we should not expose "updated" ordering which could expose recent user activity including logins. +var AdminUserOrderByMap = map[string]map[string]db.SearchOrderBy{ + "asc": { + "name": db.SearchOrderByAlphabetically, + "created": db.SearchOrderByOldest, + "updated": db.SearchOrderByLeastUpdated, + "id": db.SearchOrderByID, + }, + "desc": { + "name": db.SearchOrderByAlphabeticallyReverse, + "created": db.SearchOrderByNewest, + "updated": db.SearchOrderByRecentUpdated, + "id": db.SearchOrderByIDReverse, + }, +} + // SearchUserOptions contains the options for searching type SearchUserOptions struct { db.ListOptions diff --git a/modules/charset/charset.go b/modules/charset/charset.go index 597ce5120c..b156654973 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -5,12 +5,10 @@ package charset import ( "bytes" - "fmt" "io" "strings" "unicode/utf8" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -23,60 +21,39 @@ import ( var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} type ConvertOpts struct { - KeepBOM bool + KeepBOM bool + ErrorReplacement []byte + ErrorReturnOrigin bool } +var ToUTF8WithFallbackReaderPrefetchSize = 16 * 1024 + // ToUTF8WithFallbackReader detects the encoding of content and converts to UTF-8 reader if possible func ToUTF8WithFallbackReader(rd io.Reader, opts ConvertOpts) io.Reader { - buf := make([]byte, 2048) + buf := make([]byte, ToUTF8WithFallbackReaderPrefetchSize) n, err := util.ReadAtMost(rd, buf) if err != nil { - return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd) - } - - charsetLabel, err := DetectEncoding(buf[:n]) - if err != nil || charsetLabel == "UTF-8" { - return io.MultiReader(bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), rd) - } - - encoding, _ := charset.Lookup(charsetLabel) - if encoding == nil { + // read error occurs, don't do any processing return io.MultiReader(bytes.NewReader(buf[:n]), rd) } - return transform.NewReader( - io.MultiReader( - bytes.NewReader(MaybeRemoveBOM(buf[:n], opts)), - rd, - ), - encoding.NewDecoder(), - ) -} - -// ToUTF8 converts content to UTF8 encoding -func ToUTF8(content []byte, opts ConvertOpts) (string, error) { - charsetLabel, err := DetectEncoding(content) - if err != nil { - return "", err - } else if charsetLabel == "UTF-8" { - return string(MaybeRemoveBOM(content, opts)), nil + charsetLabel, _ := DetectEncoding(buf[:n]) + if charsetLabel == "UTF-8" { + // is utf-8, try to remove BOM and read it as-is + return io.MultiReader(bytes.NewReader(maybeRemoveBOM(buf[:n], opts)), rd) } encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { - return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel) + // unknown charset, don't do any processing + return io.MultiReader(bytes.NewReader(buf[:n]), rd) } - // If there is an error, we concatenate the nicely decoded part and the - // original left over. This way we won't lose much data. - result, n, err := transform.Bytes(encoding.NewDecoder(), content) - if err != nil { - result = append(result, content[n:]...) - } - - result = MaybeRemoveBOM(result, opts) - - return string(result), err + // convert from charset to utf-8 + return transform.NewReader( + io.MultiReader(bytes.NewReader(buf[:n]), rd), + encoding.NewDecoder(), + ) } // ToUTF8WithFallback detects the encoding of content and converts to UTF-8 if possible @@ -85,73 +62,84 @@ func ToUTF8WithFallback(content []byte, opts ConvertOpts) []byte { return bs } -// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible -func ToUTF8DropErrors(content []byte, opts ConvertOpts) []byte { - charsetLabel, err := DetectEncoding(content) - if err != nil || charsetLabel == "UTF-8" { - return MaybeRemoveBOM(content, opts) +func ToUTF8DropErrors(content []byte) []byte { + return ToUTF8(content, ConvertOpts{ErrorReplacement: []byte{' '}}) +} + +func ToUTF8(content []byte, opts ConvertOpts) []byte { + charsetLabel, _ := DetectEncoding(content) + if charsetLabel == "UTF-8" { + return maybeRemoveBOM(content, opts) } encoding, _ := charset.Lookup(charsetLabel) if encoding == nil { + setting.PanicInDevOrTesting("unsupported detected charset %q, it shouldn't happen", charsetLabel) return content } - // We ignore any non-decodable parts from the file. - // Some parts might be lost var decoded []byte decoder := encoding.NewDecoder() idx := 0 - for { + for idx < len(content) { result, n, err := transform.Bytes(decoder, content[idx:]) decoded = append(decoded, result...) if err == nil { break } - decoded = append(decoded, ' ') - idx = idx + n + 1 - if idx >= len(content) { - break + if opts.ErrorReturnOrigin { + return content } + if opts.ErrorReplacement == nil { + decoded = append(decoded, content[idx+n]) + } else { + decoded = append(decoded, opts.ErrorReplacement...) + } + idx += n + 1 } - - return MaybeRemoveBOM(decoded, opts) + return maybeRemoveBOM(decoded, opts) } -// MaybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false -func MaybeRemoveBOM(content []byte, opts ConvertOpts) []byte { +// maybeRemoveBOM removes a UTF-8 BOM from a []byte when opts.KeepBOM is false +func maybeRemoveBOM(content []byte, opts ConvertOpts) []byte { if opts.KeepBOM { return content } - if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) { - return content[3:] - } - return content + return bytes.TrimPrefix(content, UTF8BOM) } // DetectEncoding detect the encoding of content -func DetectEncoding(content []byte) (string, error) { +// it always returns a detected or guessed "encoding" string, no matter error happens or not +func DetectEncoding(content []byte) (encoding string, _ error) { // First we check if the content represents valid utf8 content excepting a truncated character at the end. // Now we could decode all the runes in turn but this is not necessarily the cheapest thing to do - // instead we walk backwards from the end to trim off a the incomplete character + // instead we walk backwards from the end to trim off the incomplete character toValidate := content end := len(toValidate) - 1 - if end < 0 { - // no-op - } else if toValidate[end]>>5 == 0b110 { - // Incomplete 1 byte extension e.g. © which has been truncated to - toValidate = toValidate[:end] - } else if end > 0 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>4 == 0b1110 { - // Incomplete 2 byte extension e.g. ⛔ <9b><94> which has been truncated to <9b> - toValidate = toValidate[:end-1] - } else if end > 1 && toValidate[end]>>6 == 0b10 && toValidate[end-1]>>6 == 0b10 && toValidate[end-2]>>3 == 0b11110 { - // Incomplete 3 byte extension e.g. 💩 <9f><92> which has been truncated to <9f><92> - toValidate = toValidate[:end-2] + // U+0000 U+007F 0yyyzzzz + // U+0080 U+07FF 110xxxyy 10yyzzzz + // U+0800 U+FFFF 1110wwww 10xxxxyy 10yyzzzz + // U+010000 U+10FFFF 11110uvv 10vvwwww 10xxxxyy 10yyzzzz + cnt := 0 + for end >= 0 && cnt < 4 { + c := toValidate[end] + if c>>5 == 0b110 || c>>4 == 0b1110 || c>>3 == 0b11110 { + // a leading byte + toValidate = toValidate[:end] + break + } else if c>>6 == 0b10 { + // a continuation byte + end-- + } else { + // not an utf-8 byte + break + } + cnt++ } + if utf8.Valid(toValidate) { - log.Debug("Detected encoding: utf-8 (fast)") return "UTF-8", nil } @@ -160,7 +148,7 @@ func DetectEncoding(content []byte) (string, error) { if len(content) < 1024 { // Check if original content is valid if _, err := textDetector.DetectBest(content); err != nil { - return "", err + return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err } times := 1024 / len(content) detectContent = make([]byte, 0, times*len(content)) @@ -171,14 +159,10 @@ func DetectEncoding(content []byte) (string, error) { detectContent = content } - // Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie break + // Now we can't use DetectBest or just results[0] because the result isn't stable - so we need a tie-break results, err := textDetector.DetectAll(detectContent) if err != nil { - if err == chardet.NotDetectedError && len(setting.Repository.AnsiCharset) > 0 { - log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) - return setting.Repository.AnsiCharset, nil - } - return "", err + return util.IfZero(setting.Repository.AnsiCharset, "UTF-8"), err } topConfidence := results[0].Confidence @@ -201,11 +185,9 @@ func DetectEncoding(content []byte) (string, error) { } // FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument - if topResult.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 { - log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) + if topResult.Charset != "UTF-8" && setting.Repository.AnsiCharset != "" { return setting.Repository.AnsiCharset, err } - log.Debug("Detected encoding: %s", topResult.Charset) - return topResult.Charset, err + return topResult.Charset, nil } diff --git a/modules/charset/charset_test.go b/modules/charset/charset_test.go index cd2e3b9aaa..0314abc347 100644 --- a/modules/charset/charset_test.go +++ b/modules/charset/charset_test.go @@ -4,108 +4,89 @@ package charset import ( - "bytes" "io" + "os" "strings" "testing" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/test" "github.com/stretchr/testify/assert" ) -func resetDefaultCharsetsOrder() { - defaultDetectedCharsetsOrder := make([]string, 0, len(setting.Repository.DetectedCharsetsOrder)) - for _, charset := range setting.Repository.DetectedCharsetsOrder { - defaultDetectedCharsetsOrder = append(defaultDetectedCharsetsOrder, strings.ToLower(strings.TrimSpace(charset))) - } +func TestMain(m *testing.M) { setting.Repository.DetectedCharsetScore = map[string]int{} - i := 0 - for _, charset := range defaultDetectedCharsetsOrder { - canonicalCharset := strings.ToLower(strings.TrimSpace(charset)) - if _, has := setting.Repository.DetectedCharsetScore[canonicalCharset]; !has { - setting.Repository.DetectedCharsetScore[canonicalCharset] = i - i++ - } + for i, charset := range setting.Repository.DetectedCharsetsOrder { + setting.Repository.DetectedCharsetScore[strings.ToLower(charset)] = i } + os.Exit(m.Run()) } func TestMaybeRemoveBOM(t *testing.T) { - res := MaybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res := maybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) - res = MaybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res = maybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) } func TestToUTF8(t *testing.T) { - resetDefaultCharsetsOrder() - // Note: golang compiler seems so behave differently depending on the current // locale, so some conversions might behave differently. For that reason, we don't // depend on particular conversions but in expected behaviors. - res, err := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, "ABC", res) + res := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) + assert.Equal(t, "ABC", string(res)) // "áéíóú" - res, err = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) + res = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // "áéíóú" - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba, }, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, []byte(res)) + assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, }, ConvertOpts{}) - assert.NoError(t, err) stringMustStartWith(t, "Hola,", res) stringMustEndWith(t, "AAA.", res) - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, }, ConvertOpts{}) - assert.NoError(t, err) stringMustStartWith(t, "Hola,", res) stringMustEndWith(t, "AAA.", res) - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, }, ConvertOpts{}) - assert.NoError(t, err) stringMustStartWith(t, "Hola,", res) stringMustEndWith(t, "AAA.", res) // Japanese (Shift-JIS) // 日属秘ぞしちゅ。 - res, err = ToUTF8([]byte{ + res = ToUTF8([]byte{ 0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42, }, ConvertOpts{}) - assert.NoError(t, err) assert.Equal(t, []byte{ 0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, 0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, - }, - []byte(res)) + }, res) - res, err = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) - assert.NoError(t, err) - assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, []byte(res)) + res = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) + assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) } func TestToUTF8WithFallback(t *testing.T) { - resetDefaultCharsetsOrder() // "ABC" res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) assert.Equal(t, []byte{0x41, 0x42, 0x43}, res) @@ -152,54 +133,58 @@ func TestToUTF8WithFallback(t *testing.T) { } func TestToUTF8DropErrors(t *testing.T) { - resetDefaultCharsetsOrder() // "ABC" - res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) + res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}) assert.Equal(t, []byte{0x41, 0x42, 0x43}, res) // "áéíóú" - res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // UTF8 BOM + "áéíóú" - res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) // "Hola, así cómo ños" - res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}) assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8]) assert.Equal(t, []byte{0x73}, res[len(res)-1:]) // "Hola, así cómo " minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20} - res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}) // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those assert.Equal(t, minmatch, res[0:len(minmatch)]) - res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}) // Do not fail for differences in invalid cases, as the library might change the conversion criteria for those assert.Equal(t, minmatch, res[0:len(minmatch)]) // Japanese (Shift-JIS) // "日属秘ぞしちゅ。" - res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}) assert.Equal(t, []byte{ 0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, 0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, }, res) - res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) + res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}) assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) } func TestDetectEncoding(t *testing.T) { - resetDefaultCharsetsOrder() testSuccess := func(b []byte, expected string) { encoding, err := DetectEncoding(b) assert.NoError(t, err) assert.Equal(t, expected, encoding) } + + // invalid bytes + encoding, err := DetectEncoding([]byte{0xfa}) + assert.Error(t, err) + assert.Equal(t, "UTF-8", encoding) + // utf-8 b := []byte("just some ascii") testSuccess(b, "UTF-8") @@ -214,169 +199,49 @@ func TestDetectEncoding(t *testing.T) { // iso-8859-1: dcor b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a} - encoding, err := DetectEncoding(b) + encoding, err = DetectEncoding(b) assert.NoError(t, err) assert.Contains(t, encoding, "ISO-8859-1") - old := setting.Repository.AnsiCharset - setting.Repository.AnsiCharset = "placeholder" - defer func() { - setting.Repository.AnsiCharset = old - }() - testSuccess(b, "placeholder") - - // invalid bytes - b = []byte{0xfa} - _, err = DetectEncoding(b) - assert.Error(t, err) + defer test.MockVariableValue(&setting.Repository.AnsiCharset, "MyEncoding")() + testSuccess(b, "MyEncoding") } -func stringMustStartWith(t *testing.T, expected, value string) { - assert.Equal(t, expected, value[:len(expected)]) +func stringMustStartWith(t *testing.T, expected string, value []byte) { + assert.Equal(t, expected, string(value[:len(expected)])) } -func stringMustEndWith(t *testing.T, expected, value string) { - assert.Equal(t, expected, value[len(value)-len(expected):]) +func stringMustEndWith(t *testing.T, expected string, value []byte) { + assert.Equal(t, expected, string(value[len(value)-len(expected):])) } func TestToUTF8WithFallbackReader(t *testing.T) { - resetDefaultCharsetsOrder() + test.MockVariableValue(&ToUTF8WithFallbackReaderPrefetchSize) - for testLen := range 2048 { - pattern := " test { () }\n" - input := "" - for len(input) < testLen { - input += pattern - } - input = input[:testLen] - input += "// Выключаем" - rd := ToUTF8WithFallbackReader(bytes.NewReader([]byte(input)), ConvertOpts{}) + block := "aá啊🤔" + runes := []rune(block) + assert.Len(t, string(runes[0]), 1) + assert.Len(t, string(runes[1]), 2) + assert.Len(t, string(runes[2]), 3) + assert.Len(t, string(runes[3]), 4) + + content := strings.Repeat(block, 2) + for i := 1; i < len(content); i++ { + encoding, err := DetectEncoding([]byte(content[:i])) + assert.NoError(t, err) + assert.Equal(t, "UTF-8", encoding) + + ToUTF8WithFallbackReaderPrefetchSize = i + rd := ToUTF8WithFallbackReader(strings.NewReader(content), ConvertOpts{}) r, _ := io.ReadAll(rd) - assert.Equalf(t, input, string(r), "testing string len=%d", testLen) + assert.Equal(t, content, string(r)) + } + for _, r := range runes { + content = "abc abc " + string(r) + string(r) + string(r) + for i := 0; i < len(content); i++ { + encoding, err := DetectEncoding([]byte(content[:i])) + assert.NoError(t, err) + assert.Equal(t, "UTF-8", encoding) + } } - - truncatedOneByteExtension := failFastBytes - encoding, _ := DetectEncoding(truncatedOneByteExtension) - assert.Equal(t, "UTF-8", encoding) - - truncatedTwoByteExtension := failFastBytes - truncatedTwoByteExtension[len(failFastBytes)-1] = 0x9b - truncatedTwoByteExtension[len(failFastBytes)-2] = 0xe2 - - encoding, _ = DetectEncoding(truncatedTwoByteExtension) - assert.Equal(t, "UTF-8", encoding) - - truncatedThreeByteExtension := failFastBytes - truncatedThreeByteExtension[len(failFastBytes)-1] = 0x92 - truncatedThreeByteExtension[len(failFastBytes)-2] = 0x9f - truncatedThreeByteExtension[len(failFastBytes)-3] = 0xf0 - - encoding, _ = DetectEncoding(truncatedThreeByteExtension) - assert.Equal(t, "UTF-8", encoding) -} - -var failFastBytes = []byte{ - 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x74, 0x6f, - 0x6f, 0x6c, 0x73, 0x2e, 0x61, 0x6e, 0x74, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x64, 0x65, 0x66, 0x73, 0x2e, 0x63, 0x6f, 0x6e, - 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4f, 0x73, 0x0a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x20, 0x6f, 0x72, 0x67, - 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, - 0x74, 0x2e, 0x67, 0x72, 0x61, 0x64, 0x6c, 0x65, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x75, 0x6e, 0x2e, 0x42, - 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x0a, 0x0a, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x64, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, - 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, - 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x61, 0x70, 0x69, 0x2d, 0x64, 0x6f, 0x63, 0x73, 0x22, 0x29, - 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x64, 0x62, - 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, - 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x66, - 0x73, 0x22, 0x29, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x28, 0x22, 0x3a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x3a, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x6d, 0x71, 0x22, 0x29, 0x29, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, - 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, - 0x2d, 0x61, 0x75, 0x74, 0x68, 0x2d, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x65, 0x72, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, - 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x68, 0x61, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, - 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x29, 0x0a, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, - 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, - 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x77, 0x65, 0x62, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, - 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, - 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x6f, 0x70, - 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, - 0x72, 0x6b, 0x2e, 0x62, 0x6f, 0x6f, 0x74, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x2d, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x63, 0x74, 0x75, 0x61, 0x74, 0x6f, 0x72, 0x22, 0x29, 0x0a, 0x20, - 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, - 0x72, 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x72, 0x2d, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x22, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, - 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2d, 0x61, 0x6c, 0x6c, 0x22, 0x29, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, - 0x67, 0x2e, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x63, 0x6c, - 0x6f, 0x75, 0x64, 0x3a, 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2d, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x65, 0x72, 0x2d, 0x73, 0x6c, 0x65, 0x75, 0x74, 0x68, 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, - 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6f, 0x72, 0x67, 0x2e, 0x73, 0x70, - 0x72, 0x69, 0x6e, 0x67, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x77, 0x6f, 0x72, 0x6b, 0x2e, 0x72, 0x65, 0x74, 0x72, 0x79, 0x3a, - 0x73, 0x70, 0x72, 0x69, 0x6e, 0x67, 0x2d, 0x72, 0x65, 0x74, 0x72, 0x79, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x63, 0x68, 0x2e, 0x71, - 0x6f, 0x73, 0x2e, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x3a, 0x6c, 0x6f, 0x67, 0x62, 0x61, 0x63, 0x6b, 0x2d, 0x63, - 0x6c, 0x61, 0x73, 0x73, 0x69, 0x63, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, - 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x69, 0x6f, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65, - 0x74, 0x65, 0x72, 0x3a, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x2d, 0x72, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x72, 0x79, 0x2d, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x6b, 0x6f, 0x74, - 0x6c, 0x69, 0x6e, 0x28, 0x22, 0x73, 0x74, 0x64, 0x6c, 0x69, 0x62, 0x22, 0x29, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, - 0x2f, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0x54, 0x65, 0x73, 0x74, 0x20, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, - 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x2e, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, - 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x2f, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x74, - 0x65, 0x73, 0x74, 0x49, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x28, 0x22, 0x6a, - 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x65, 0x3a, 0x70, 0x65, 0x2d, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2d, - 0x74, 0x65, 0x73, 0x74, 0x22, 0x29, 0x0a, 0x7d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a, - 0x61, 0x72, 0x20, 0x62, 0x79, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x2e, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, - 0x69, 0x6e, 0x67, 0x28, 0x4a, 0x61, 0x72, 0x3a, 0x3a, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, - 0x20, 0x20, 0x61, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x69, 0x65, 0x72, 0x2e, - 0x73, 0x65, 0x74, 0x28, 0x22, 0x70, 0x61, 0x74, 0x63, 0x68, 0x65, 0x64, 0x22, 0x29, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, - 0x76, 0x61, 0x6c, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68, - 0x20, 0x62, 0x79, 0x20, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x67, - 0x65, 0x74, 0x74, 0x69, 0x6e, 0x67, 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, - 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, - 0x73, 0x28, 0x22, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x2d, 0x50, 0x61, 0x74, 0x68, 0x22, 0x20, 0x74, 0x6f, 0x20, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x70, - 0x72, 0x69, 0x76, 0x61, 0x74, 0x65, 0x20, 0x76, 0x61, 0x6c, 0x20, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x20, 0x3d, - 0x20, 0x22, 0x66, 0x69, 0x6c, 0x65, 0x3a, 0x2f, 0x2b, 0x22, 0x2e, 0x74, 0x6f, 0x52, 0x65, 0x67, 0x65, 0x78, 0x28, 0x29, - 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, - 0x65, 0x20, 0x66, 0x75, 0x6e, 0x20, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x3a, 0x20, 0x53, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x20, 0x3d, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x70, - 0x61, 0x74, 0x68, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x2e, 0x6a, 0x6f, 0x69, 0x6e, 0x54, 0x6f, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x28, 0x22, 0x20, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x69, 0x74, 0x2e, 0x74, 0x6f, 0x55, 0x52, 0x49, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x55, - 0x52, 0x4c, 0x28, 0x29, 0x2e, 0x74, 0x6f, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x28, 0x29, 0x2e, 0x72, 0x65, 0x70, 0x6c, - 0x61, 0x63, 0x65, 0x46, 0x69, 0x72, 0x73, 0x74, 0x28, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x2c, 0x20, 0x22, 0x2f, - 0x22, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x20, 0x20, 0x20, - 0x20, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x7d, 0x0a, 0x0a, 0x74, 0x61, 0x73, - 0x6b, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x3c, 0x42, 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x3e, 0x28, 0x22, 0x62, - 0x6f, 0x6f, 0x74, 0x52, 0x75, 0x6e, 0x22, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69, 0x66, 0x20, 0x28, 0x4f, - 0x73, 0x2e, 0x69, 0x73, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x28, 0x4f, 0x73, 0x2e, 0x46, 0x41, 0x4d, 0x49, 0x4c, 0x59, - 0x5f, 0x57, 0x49, 0x4e, 0x44, 0x4f, 0x57, 0x53, 0x29, 0x29, 0x20, 0x7b, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, - 0x20, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x70, 0x61, 0x74, 0x68, 0x20, 0x3d, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x28, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x53, 0x65, 0x74, 0x73, 0x2e, 0x6e, 0x61, 0x6d, 0x65, 0x64, 0x28, 0x22, 0x6d, 0x61, 0x69, - 0x6e, 0x22, 0x29, 0x2e, 0x6d, 0x61, 0x70, 0x20, 0x7b, 0x20, 0x69, 0x74, 0x2e, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x20, - 0x7d, 0x2c, 0x20, 0x70, 0x61, 0x74, 0x63, 0x68, 0x4a, 0x61, 0x72, 0x29, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x7d, 0x0a, 0x0a, - 0x20, 0x20, 0x20, 0x20, 0x2f, 0x2f, 0x20, 0xd0, } diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go index b4c5e7fe1e..2d66a86a8b 100644 --- a/modules/httplib/serve.go +++ b/modules/httplib/serve.go @@ -19,7 +19,6 @@ import ( charsetModule "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/container" "code.gitea.io/gitea/modules/httpcache" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" @@ -109,11 +108,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, mineBuf []byt } if isPlain { - charset, err := charsetModule.DetectEncoding(mineBuf) - if err != nil { - log.Error("Detect raw file %s charset failed: %v, using by default utf-8", opts.Filename, err) - charset = "utf-8" - } + charset, _ := charsetModule.DetectEncoding(mineBuf) opts.ContentTypeCharset = strings.ToLower(charset) } diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index bdb477ce6e..5f6a7f6082 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -203,7 +203,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro RepoID: repo.ID, CommitID: commitSha, Filename: update.Filename, - Content: string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), + Content: string(charset.ToUTF8DropErrors(fileContents)), Language: analyze.GetCodeLanguage(update.Filename, fileContents), UpdatedAt: time.Now().UTC(), }) diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index b2eb301a5d..a7027051d2 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -191,7 +191,7 @@ func (b *Indexer) addUpdate(ctx context.Context, batchWriter git.WriteCloserErro Doc(map[string]any{ "repo_id": repo.ID, "filename": update.Filename, - "content": string(charset.ToUTF8DropErrors(fileContents, charset.ConvertOpts{})), + "content": string(charset.ToUTF8DropErrors(fileContents)), "commit_id": sha, "language": analyze.GetCodeLanguage(update.Filename, fileContents), "updated_at": timeutil.TimeStampNow(), diff --git a/modules/packages/npm/creator.go b/modules/packages/npm/creator.go index 11b5123c27..cc7695726b 100644 --- a/modules/packages/npm/creator.go +++ b/modules/packages/npm/creator.go @@ -62,7 +62,28 @@ type PackageMetadata struct { Author User `json:"author"` ReadmeFilename string `json:"readmeFilename,omitempty"` Users map[string]bool `json:"users,omitempty"` - License string `json:"license,omitempty"` + License License `json:"license,omitempty"` +} + +type License string + +func (l *License) UnmarshalJSON(data []byte) error { + switch data[0] { + case '"': + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + *l = License(value) + case '{': + var values map[string]any + if err := json.Unmarshal(data, &values); err != nil { + return err + } + value, _ := values["type"].(string) + *l = License(value) + } + return nil } // PackageMetadataVersion documentation: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version @@ -74,7 +95,7 @@ type PackageMetadataVersion struct { Description string `json:"description"` Author User `json:"author"` Homepage string `json:"homepage,omitempty"` - License string `json:"license,omitempty"` + License License `json:"license,omitempty"` Repository Repository `json:"repository"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` diff --git a/modules/packages/npm/creator_test.go b/modules/packages/npm/creator_test.go index 806377a52b..40c50de91f 100644 --- a/modules/packages/npm/creator_test.go +++ b/modules/packages/npm/creator_test.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/json" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestParsePackage(t *testing.T) { @@ -291,11 +292,36 @@ func TestParsePackage(t *testing.T) { assert.Equal(t, packageDescription, p.Metadata.Readme) assert.Equal(t, packageAuthor, p.Metadata.Author) assert.Equal(t, packageBin, p.Metadata.Bin["bin"]) - assert.Equal(t, "MIT", p.Metadata.License) + assert.Equal(t, "MIT", string(p.Metadata.License)) assert.Equal(t, "https://gitea.io/", p.Metadata.ProjectURL) assert.Contains(t, p.Metadata.Dependencies, "package") assert.Equal(t, "1.2.0", p.Metadata.Dependencies["package"]) assert.Equal(t, repository.Type, p.Metadata.Repository.Type) assert.Equal(t, repository.URL, p.Metadata.Repository.URL) }) + + t.Run("ValidLicenseMap", func(t *testing.T) { + packageJSON := `{ + "versions": { + "0.1.1": { + "name": "dev-null", + "version": "0.1.1", + "license": { + "type": "MIT" + }, + "dist": { + "integrity": "sha256-" + } + } + }, + "_attachments": { + "foo": { + "data": "AAAA" + } + } +}` + p, err := ParsePackage(strings.NewReader(packageJSON)) + require.NoError(t, err) + require.Equal(t, "MIT", string(p.Metadata.License)) + }) } diff --git a/modules/packages/npm/metadata.go b/modules/packages/npm/metadata.go index 362d0470d5..e6bbcb1177 100644 --- a/modules/packages/npm/metadata.go +++ b/modules/packages/npm/metadata.go @@ -12,7 +12,7 @@ type Metadata struct { Name string `json:"name,omitempty"` Description string `json:"description,omitempty"` Author string `json:"author,omitempty"` - License string `json:"license,omitempty"` + License License `json:"license,omitempty"` ProjectURL string `json:"project_url,omitempty"` Keywords []string `json:"keywords,omitempty"` Dependencies map[string]string `json:"dependencies,omitempty"` diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e14997801f..dc60d99bd6 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -240,4 +240,5 @@ func PanicInDevOrTesting(msg string, a ...any) { if !IsProd || IsInTesting { panic(fmt.Sprintf(msg, a...)) } + log.Error(msg, a...) } diff --git a/routers/api/v1/admin/user.go b/routers/api/v1/admin/user.go index 6f1e2eb120..6bed410642 100644 --- a/routers/api/v1/admin/user.go +++ b/routers/api/v1/admin/user.go @@ -414,22 +414,116 @@ func SearchUsers(ctx *context.APIContext) { // in: query // description: page size of results // type: integer + // - name: sort + // in: query + // description: sort users by attribute. Supported values are + // "name", "created", "updated" and "id". + // Default is "name" + // type: string + // - name: order + // in: query + // description: sort order, either "asc" (ascending) or "desc" (descending). + // Default is "asc", ignored if "sort" is not specified. + // type: string + // - name: q + // in: query + // description: search term (username, full name, email) + // type: string + // - name: visibility + // in: query + // description: visibility filter. Supported values are + // "public", "limited" and "private". + // type: string + // - name: is_active + // in: query + // description: filter active users + // type: boolean + // - name: is_admin + // in: query + // description: filter admin users + // type: boolean + // - name: is_restricted + // in: query + // description: filter restricted users + // type: boolean + // - name: is_2fa_enabled + // in: query + // description: filter 2FA enabled users + // type: boolean + // - name: is_prohibit_login + // in: query + // description: filter login prohibited users + // type: boolean // responses: // "200": // "$ref": "#/responses/UserList" // "403": // "$ref": "#/responses/forbidden" + // "422": + // "$ref": "#/responses/validationError" listOptions := utils.GetListOptions(ctx) - users, maxResults, err := user_model.SearchUsers(ctx, user_model.SearchUserOptions{ - Actor: ctx.Doer, - Types: []user_model.UserType{user_model.UserTypeIndividual}, - LoginName: ctx.FormTrim("login_name"), - SourceID: ctx.FormInt64("source_id"), - OrderBy: db.SearchOrderByAlphabetically, - ListOptions: listOptions, - }) + orderBy := db.SearchOrderByAlphabetically + sortMode := ctx.FormString("sort") + if len(sortMode) > 0 { + sortOrder := ctx.FormString("order") + if len(sortOrder) == 0 { + sortOrder = "asc" + } + if searchModeMap, ok := user_model.AdminUserOrderByMap[sortOrder]; ok { + if order, ok := searchModeMap[sortMode]; ok { + orderBy = order + } else { + ctx.APIError(http.StatusUnprocessableEntity, fmt.Errorf("Invalid sort mode: \"%s\"", sortMode)) + return + } + } else { + ctx.APIError(http.StatusUnprocessableEntity, fmt.Errorf("Invalid sort order: \"%s\"", sortOrder)) + return + } + } + + var visible []api.VisibleType + visibilityParam := ctx.FormString("visibility") + if len(visibilityParam) > 0 { + if visibility, ok := api.VisibilityModes[visibilityParam]; ok { + visible = []api.VisibleType{visibility} + } else { + ctx.APIError(http.StatusUnprocessableEntity, fmt.Errorf("Invalid visibility: \"%s\"", visibilityParam)) + return + } + } + + searchOpts := user_model.SearchUserOptions{ + Actor: ctx.Doer, + Types: []user_model.UserType{user_model.UserTypeIndividual}, + LoginName: ctx.FormTrim("login_name"), + SourceID: ctx.FormInt64("source_id"), + Keyword: ctx.FormTrim("q"), + Visible: visible, + OrderBy: orderBy, + ListOptions: listOptions, + SearchByEmail: true, + } + + if ctx.FormString("is_active") != "" { + searchOpts.IsActive = optional.Some(ctx.FormBool("is_active")) + } + if ctx.FormString("is_admin") != "" { + searchOpts.IsAdmin = optional.Some(ctx.FormBool("is_admin")) + } + if ctx.FormString("is_restricted") != "" { + searchOpts.IsRestricted = optional.Some(ctx.FormBool("is_restricted")) + } + if ctx.FormString("is_2fa_enabled") != "" { + searchOpts.IsTwoFactorEnabled = optional.Some(ctx.FormBool("is_2fa_enabled")) + } + if ctx.FormString("is_prohibit_login") != "" { + searchOpts.IsProhibitLogin = optional.Some(ctx.FormBool("is_prohibit_login")) + } + + users, maxResults, err := user_model.SearchUsers(ctx, searchOpts) if err != nil { ctx.APIErrorInternal(err) return diff --git a/routers/web/repo/editor.go b/routers/web/repo/editor.go index 983249a6d2..048c9f3d4a 100644 --- a/routers/web/repo/editor.go +++ b/routers/web/repo/editor.go @@ -317,11 +317,7 @@ func EditFile(ctx *context.Context) { ctx.ServerError("ReadAll", err) return } - if content, err := charset.ToUTF8(buf, charset.ConvertOpts{KeepBOM: true}); err != nil { - ctx.Data["FileContent"] = string(buf) - } else { - ctx.Data["FileContent"] = content - } + ctx.Data["FileContent"] = string(charset.ToUTF8(buf, charset.ConvertOpts{KeepBOM: true, ErrorReturnOrigin: true})) } } diff --git a/routers/web/repo/editor_cherry_pick.go b/routers/web/repo/editor_cherry_pick.go index c1f3ae861b..ca0e19517a 100644 --- a/routers/web/repo/editor_cherry_pick.go +++ b/routers/web/repo/editor_cherry_pick.go @@ -36,9 +36,7 @@ func CherryPick(ctx *context.Context) { ctx.Data["commit_message"] = "revert " + cherryPickCommit.Message() } else { ctx.Data["CherryPickType"] = "cherry-pick" - splits := strings.SplitN(cherryPickCommit.Message(), "\n", 2) - ctx.Data["commit_summary"] = splits[0] - ctx.Data["commit_message"] = splits[1] + ctx.Data["commit_summary"], ctx.Data["commit_message"], _ = strings.Cut(cherryPickCommit.Message(), "\n") } ctx.HTML(http.StatusOK, tplCherryPick) diff --git a/services/gitdiff/gitdiff.go b/services/gitdiff/gitdiff.go index 6e15f71609..f8fde6ab29 100644 --- a/services/gitdiff/gitdiff.go +++ b/services/gitdiff/gitdiff.go @@ -835,11 +835,11 @@ parsingLoop: if buffer.Len() == 0 { continue } - charsetLabel, err := charset.DetectEncoding(buffer.Bytes()) - if charsetLabel != "UTF-8" && err == nil { - encoding, _ := stdcharset.Lookup(charsetLabel) - if encoding != nil { - diffLineTypeDecoders[lineType] = encoding.NewDecoder() + charsetLabel, _ := charset.DetectEncoding(buffer.Bytes()) + if charsetLabel != "UTF-8" { + charsetEncoding, _ := stdcharset.Lookup(charsetLabel) + if charsetEncoding != nil { + diffLineTypeDecoders[lineType] = charsetEncoding.NewDecoder() } } } @@ -1325,10 +1325,10 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit shouldFullFileHighlight := !setting.Git.DisableDiffHighlight && attrDiff.Value() == "" if shouldFullFileHighlight { if limitedContent.LeftContent != nil && limitedContent.LeftContent.buf.Len() < MaxDiffHighlightEntireFileSize { - diffFile.highlightedLeftLines = highlightCodeLines(diffFile, true /* left */, limitedContent.LeftContent.buf.String()) + diffFile.highlightedLeftLines = highlightCodeLines(diffFile, true /* left */, limitedContent.LeftContent.buf.Bytes()) } if limitedContent.RightContent != nil && limitedContent.RightContent.buf.Len() < MaxDiffHighlightEntireFileSize { - diffFile.highlightedRightLines = highlightCodeLines(diffFile, false /* right */, limitedContent.RightContent.buf.String()) + diffFile.highlightedRightLines = highlightCodeLines(diffFile, false /* right */, limitedContent.RightContent.buf.Bytes()) } } } @@ -1336,9 +1336,34 @@ func GetDiffForRender(ctx context.Context, repoLink string, gitRepo *git.Reposit return diff, nil } -func highlightCodeLines(diffFile *DiffFile, isLeft bool, content string) map[int]template.HTML { +func splitHighlightLines(buf []byte) (ret [][]byte) { + lineCount := bytes.Count(buf, []byte("\n")) + 1 + ret = make([][]byte, 0, lineCount) + nlTagClose := []byte("\n" right after \n, sometimes before. + // * "text\n" + // * "text\n" + if bytes.HasPrefix(buf[pos:], nlTagClose) { + pos1 := bytes.IndexByte(buf[pos:], '>') + if pos1 != -1 { + pos += pos1 + } + } + ret = append(ret, buf[:pos+1]) + buf = buf[pos+1:] + } +} + +func highlightCodeLines(diffFile *DiffFile, isLeft bool, rawContent []byte) map[int]template.HTML { + content := util.UnsafeBytesToString(charset.ToUTF8(rawContent, charset.ConvertOpts{})) highlightedNewContent, _ := highlight.Code(diffFile.Name, diffFile.Language, content) - splitLines := strings.Split(string(highlightedNewContent), "\n") + splitLines := splitHighlightLines([]byte(highlightedNewContent)) lines := make(map[int]template.HTML, len(splitLines)) // only save the highlighted lines we need, but not the whole file, to save memory for _, sec := range diffFile.Sections { diff --git a/services/gitdiff/gitdiff_test.go b/services/gitdiff/gitdiff_test.go index 721ae0dfc7..a94dad8b63 100644 --- a/services/gitdiff/gitdiff_test.go +++ b/services/gitdiff/gitdiff_test.go @@ -5,6 +5,7 @@ package gitdiff import ( + "html/template" "strconv" "strings" "testing" @@ -1106,3 +1107,41 @@ func TestDiffLine_GetExpandDirection(t *testing.T) { assert.Equal(t, c.direction, c.diffLine.GetExpandDirection(), "case %s expected direction: %s", c.name, c.direction) } } + +func TestHighlightCodeLines(t *testing.T) { + t.Run("CharsetDetecting", func(t *testing.T) { + diffFile := &DiffFile{ + Name: "a.c", + Language: "c", + Sections: []*DiffSection{ + { + Lines: []*DiffLine{{LeftIdx: 1}}, + }, + }, + } + ret := highlightCodeLines(diffFile, true, []byte("// abc\xcc def\xcd")) // ISO-8859-1 bytes + assert.Equal(t, "// abcÌ defÍ\n", string(ret[0])) + }) + + t.Run("LeftLines", func(t *testing.T) { + diffFile := &DiffFile{ + Name: "a.c", + Language: "c", + Sections: []*DiffSection{ + { + Lines: []*DiffLine{ + {LeftIdx: 1}, + {LeftIdx: 2}, + {LeftIdx: 3}, + }, + }, + }, + } + const nl = "\n" + ret := highlightCodeLines(diffFile, true, []byte("a\nb\n")) + assert.Equal(t, map[int]template.HTML{ + 0: `a` + nl, + 1: `b`, + }, ret) + }) +} diff --git a/services/gitdiff/highlightdiff_test.go b/services/gitdiff/highlightdiff_test.go index aebe38ae7c..0df2e29d13 100644 --- a/services/gitdiff/highlightdiff_test.go +++ b/services/gitdiff/highlightdiff_test.go @@ -25,12 +25,12 @@ func TestDiffWithHighlight(t *testing.T) { t.Run("CleanUp", func(t *testing.T) { hcd := newHighlightCodeDiff() - codeA := template.HTML(`this is updated comment`) + codeA := template.HTML(`this is a comment`) + codeB := template.HTML(`this is updated comment`) outDel := hcd.diffLineWithHighlight(DiffLineDel, codeA, codeB) - assert.Equal(t, `a comment`, string(outDel)) + assert.Equal(t, `this is a comment`, string(outDel)) outAdd := hcd.diffLineWithHighlight(DiffLineAdd, codeA, codeB) - assert.Equal(t, `updated comment`, string(outAdd)) + assert.Equal(t, `this is updated comment`, string(outAdd)) }) t.Run("OpenCloseTags", func(t *testing.T) { diff --git a/services/mailer/sender/sender.go b/services/mailer/sender/sender.go index e470c2f2b3..30c6feaf7a 100644 --- a/services/mailer/sender/sender.go +++ b/services/mailer/sender/sender.go @@ -4,10 +4,8 @@ package sender import ( + "errors" "io" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" ) type Sender interface { @@ -16,23 +14,18 @@ type Sender interface { var Send = send -func send(sender Sender, msgs ...*Message) error { - if setting.MailService == nil { - log.Error("Mailer: Send is being invoked but mail service hasn't been initialized") - return nil +func send(sender Sender, msg *Message) error { + m := msg.ToMessage() + froms := m.GetFrom() + to, err := m.GetRecipients() + if err != nil { + return err } - for _, msg := range msgs { - m := msg.ToMessage() - froms := m.GetFrom() - to, err := m.GetRecipients() - if err != nil { - return err - } - // TODO: implement sending from multiple addresses - if err := sender.Send(froms[0].Address, to, m); err != nil { - return err - } + // TODO: implement sending from multiple addresses + if len(froms) == 0 { + // FIXME: no idea why sometimes the "froms" can be empty, need to figure out the root problem + return errors.New("no FROM specified") } - return nil + return sender.Send(froms[0].Address, to, m) } diff --git a/services/repository/generate.go b/services/repository/generate.go index 3ec31dac22..b2913cd110 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -177,7 +177,7 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t } generatedContent := generateExpansion(ctx, string(content), templateRepo, generateRepo) - substSubPath := filepath.Clean(filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo))) + substSubPath := filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo)) newLocalPath := filepath.Join(tmpDir, substSubPath) regular, err := util.IsRegularFile(newLocalPath) if canWrite := regular || errors.Is(err, fs.ErrNotExist); !canWrite { @@ -358,5 +358,5 @@ func filePathSanitize(s string) string { } fields[i] = field } - return filepath.FromSlash(strings.Join(fields, "/")) + return filepath.Clean(filepath.FromSlash(strings.Trim(strings.Join(fields, "/"), "/"))) } diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index 9c01911ded..432de4dc59 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -54,19 +54,24 @@ text/*.txt } func TestFilePathSanitize(t *testing.T) { - assert.Equal(t, "test_CON", filePathSanitize("test_CON")) - assert.Equal(t, "test CON", filePathSanitize("test CON ")) - assert.Equal(t, "__/traverse/__", filePathSanitize(".. /traverse/ ..")) - assert.Equal(t, "./__/a/_git/b_", filePathSanitize("./../a/.git/ b: ")) + // path clean + assert.Equal(t, "a", filePathSanitize("//a/")) + assert.Equal(t, "_a", filePathSanitize(`\a`)) + assert.Equal(t, "__/a/__", filePathSanitize(".. /a/ ..")) + assert.Equal(t, "__/a/_git/b_", filePathSanitize("./../a/.git/ b: ")) + + // Windows reserved names assert.Equal(t, "_", filePathSanitize("CoN")) assert.Equal(t, "_", filePathSanitize("LpT1")) assert.Equal(t, "_", filePathSanitize("CoM1")) + assert.Equal(t, "test_CON", filePathSanitize("test_CON")) + assert.Equal(t, "test CON", filePathSanitize("test CON ")) + + // special chars assert.Equal(t, "_", filePathSanitize("\u0000")) - assert.Equal(t, "目标", filePathSanitize("目标")) - // unlike filepath.Clean, it only sanitizes, doesn't change the separator layout - assert.Equal(t, "", filePathSanitize("")) //nolint:testifylint // for easy reading + assert.Equal(t, ".", filePathSanitize("")) assert.Equal(t, ".", filePathSanitize(".")) - assert.Equal(t, "/", filePathSanitize("/")) + assert.Equal(t, ".", filePathSanitize("/")) } func TestProcessGiteaTemplateFile(t *testing.T) { diff --git a/templates/package/content/pypi.tmpl b/templates/package/content/pypi.tmpl index 2625c160fe..15d8971eaa 100644 --- a/templates/package/content/pypi.tmpl +++ b/templates/package/content/pypi.tmpl @@ -4,7 +4,7 @@
- 
pip install --index-url  --extra-index-url https://pypi.org/ {{.PackageDescriptor.Package.Name}}
+ 
pip install --index-url  --extra-index-url https://pypi.org/simple {{.PackageDescriptor.Package.Name}}
diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 089cdf2ccd..6d23186d08 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -96,7 +96,7 @@
{{else if eq .Type 2}}
- {{svg "octicon-circle-slash"}} + {{svg "octicon-issue-closed"}} {{if not .OriginalAuthor}} {{template "shared/user/avatarlink" dict "user" .Poster}} {{end}} diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 8fce1b6f2c..809b1e9677 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -62,7 +62,7 @@ {{if not .IsDisplayingSource}}data-raw-file-link="{{$.RawFileLink}}"{{end}} data-tooltip-content="{{if .CanCopyContent}}{{ctx.Locale.Tr "copy_content"}}{{else}}{{ctx.Locale.Tr "copy_type_unsupported"}}{{end}}" >{{svg "octicon-copy"}} - {{if .EnableFeed}} + {{if and .EnableFeed .RefFullName.IsBranch}} {{svg "octicon-rss"}} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index b37937dcee..056e05ae4d 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -781,6 +781,60 @@ "description": "page size of results", "name": "limit", "in": "query" + }, + { + "type": "string", + "description": "sort users by attribute. Supported values are \"name\", \"created\", \"updated\" and \"id\". Default is \"name\"", + "name": "sort", + "in": "query" + }, + { + "type": "string", + "description": "sort order, either \"asc\" (ascending) or \"desc\" (descending). Default is \"asc\", ignored if \"sort\" is not specified.", + "name": "order", + "in": "query" + }, + { + "type": "string", + "description": "search term (username, full name, email)", + "name": "q", + "in": "query" + }, + { + "type": "string", + "description": "visibility filter. Supported values are \"public\", \"limited\" and \"private\".", + "name": "visibility", + "in": "query" + }, + { + "type": "boolean", + "description": "filter active users", + "name": "is_active", + "in": "query" + }, + { + "type": "boolean", + "description": "filter admin users", + "name": "is_admin", + "in": "query" + }, + { + "type": "boolean", + "description": "filter restricted users", + "name": "is_restricted", + "in": "query" + }, + { + "type": "boolean", + "description": "filter 2FA enabled users", + "name": "is_2fa_enabled", + "in": "query" + }, + { + "type": "boolean", + "description": "filter login prohibited users", + "name": "is_prohibit_login", + "in": "query" } ], "responses": { @@ -789,6 +843,9 @@ }, "403": { "$ref": "#/responses/forbidden" + }, + "422": { + "$ref": "#/responses/validationError" } } }, diff --git a/tests/integration/migration-test/migration_test.go b/tests/integration/migration-test/migration_test.go index 5fa7cbbfb7..2659c5c53d 100644 --- a/tests/integration/migration-test/migration_test.go +++ b/tests/integration/migration-test/migration_test.go @@ -4,6 +4,7 @@ package migrations import ( + "bytes" "compress/gzip" "context" "database/sql" @@ -21,7 +22,6 @@ import ( "code.gitea.io/gitea/models/migrations" migrate_base "code.gitea.io/gitea/models/migrations/base" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -108,11 +108,11 @@ func readSQLFromFile(version string) (string, error) { } defer gr.Close() - bytes, err := io.ReadAll(gr) + buf, err := io.ReadAll(gr) if err != nil { return "", err } - return string(charset.MaybeRemoveBOM(bytes, charset.ConvertOpts{})), nil + return string(bytes.TrimPrefix(buf, []byte{'\xef', '\xbb', '\xbf'})), nil } func restoreOldDB(t *testing.T, version string) { diff --git a/web_src/css/base.css b/web_src/css/base.css index be28cd6fea..0e690a0265 100644 --- a/web_src/css/base.css +++ b/web_src/css/base.css @@ -39,6 +39,8 @@ --gap-inline: 0.25rem; /* gap for inline texts and elements, for example: the spaces for sentence with labels, button text, etc */ --gap-block: 0.5rem; /* gap for element blocks, for example: spaces between buttons, menu image & title, header icon & title etc */ + + --background-view-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAG0lEQVQYlWN4+vTpf3SMDTAMBYXYBLFpHgoKAeiOf0SGE9kbAAAAAElFTkSuQmCC") right bottom var(--color-primary-light-7); } @media (min-width: 768px) and (max-width: 1200px) { diff --git a/web_src/css/features/imagediff.css b/web_src/css/features/imagediff.css index ad3165e8d8..d32a2098ca 100644 --- a/web_src/css/features/imagediff.css +++ b/web_src/css/features/imagediff.css @@ -13,7 +13,7 @@ .image-diff-container img { border: 1px solid var(--color-primary-light-7); - background: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAG0lEQVQYlWN4+vTpf3SMDTAMBYXYBLFpHgoKAeiOf0SGE9kbAAAAAElFTkSuQmCC") right bottom var(--color-primary-light-7); + background: var(--background-view-image); } .image-diff-container .before-container { diff --git a/web_src/css/repo/file-view.css b/web_src/css/repo/file-view.css index 907f136afe..3f1c42a4a1 100644 --- a/web_src/css/repo/file-view.css +++ b/web_src/css/repo/file-view.css @@ -81,6 +81,7 @@ .view-raw img[src$=".svg" i] { max-height: 600px !important; max-width: 600px !important; + background: var(--background-view-image); } .file-view-render-container { diff --git a/web_src/js/components/ViewFileTreeItem.vue b/web_src/js/components/ViewFileTreeItem.vue index 9a50adedaa..ce019e655f 100644 --- a/web_src/js/components/ViewFileTreeItem.vue +++ b/web_src/js/components/ViewFileTreeItem.vue @@ -2,20 +2,10 @@ import {SvgIcon} from '../svg.ts'; import {isPlainClick} from '../utils/dom.ts'; import {shallowRef} from 'vue'; -import {type createViewFileTreeStore} from './ViewFileTreeStore.ts'; - -export type Item = { - entryName: string; - entryMode: 'blob' | 'exec' | 'tree' | 'commit' | 'symlink' | 'unknown'; - entryIcon: string; - entryIconOpen: string; - fullPath: string; - submoduleUrl?: string; - children?: Item[]; -}; +import type {createViewFileTreeStore, FileTreeItem} from './ViewFileTreeStore.ts'; const props = defineProps<{ - item: Item, + item: FileTreeItem, store: ReturnType }>(); diff --git a/web_src/js/components/ViewFileTreeStore.ts b/web_src/js/components/ViewFileTreeStore.ts index 2dc8093878..936db07776 100644 --- a/web_src/js/components/ViewFileTreeStore.ts +++ b/web_src/js/components/ViewFileTreeStore.ts @@ -3,11 +3,20 @@ import {GET} from '../modules/fetch.ts'; import {pathEscapeSegments} from '../utils/url.ts'; import {createElementFromHTML} from '../utils/dom.ts'; import {html} from '../utils/html.ts'; -import type {Item} from './ViewFileTreeItem.vue'; + +export type FileTreeItem = { + entryName: string; + entryMode: 'blob' | 'exec' | 'tree' | 'commit' | 'symlink' | 'unknown'; + entryIcon: string; + entryIconOpen: string; + fullPath: string; + submoduleUrl?: string; + children?: Array; +}; export function createViewFileTreeStore(props: {repoLink: string, treePath: string, currentRefNameSubURL: string}) { const store = reactive({ - rootFiles: [] as Array, + rootFiles: [] as Array, selectedItem: props.treePath, async loadChildren(treePath: string, subPath: string = '') { diff --git a/web_src/js/features/imagediff.ts b/web_src/js/features/imagediff.ts index 4ace1ca2ad..23f05fbdc7 100644 --- a/web_src/js/features/imagediff.ts +++ b/web_src/js/features/imagediff.ts @@ -3,7 +3,33 @@ import {hideElem, loadElem, queryElemChildren, queryElems} from '../utils/dom.ts import {parseDom} from '../utils.ts'; import {fomanticQuery} from '../modules/fomantic/base.ts'; -function getDefaultSvgBoundsIfUndefined(text: string, src: string) { +type ImageContext = { + imageBefore: HTMLImageElement | undefined, + imageAfter: HTMLImageElement | undefined, + sizeBefore: {width: number, height: number}, + sizeAfter: {width: number, height: number}, + maxSize: {width: number, height: number}, + ratio: [number, number, number, number], +}; + +type ImageInfo = { + path: string | null, + mime: string | null, + images: NodeListOf, + boundsInfo: HTMLElement | null, +}; + +type Bounds = { + width: number, + height: number, +} | null; + +type SvgBoundsInfo = { + before: Bounds, + after: Bounds, +}; + +function getDefaultSvgBoundsIfUndefined(text: string, src: string): Bounds | null { const defaultSize = 300; const maxSize = 99999; @@ -38,14 +64,14 @@ function getDefaultSvgBoundsIfUndefined(text: string, src: string) { return null; } -function createContext(imageAfter: HTMLImageElement, imageBefore: HTMLImageElement) { +function createContext(imageAfter: HTMLImageElement, imageBefore: HTMLImageElement, svgBoundsInfo: SvgBoundsInfo): ImageContext { const sizeAfter = { - width: imageAfter?.width || 0, - height: imageAfter?.height || 0, + width: svgBoundsInfo.after?.width || imageAfter?.width || 0, + height: svgBoundsInfo.after?.height || imageAfter?.height || 0, }; const sizeBefore = { - width: imageBefore?.width || 0, - height: imageBefore?.height || 0, + width: svgBoundsInfo.before?.width || imageBefore?.width || 0, + height: svgBoundsInfo.before?.height || imageBefore?.height || 0, }; const maxSize = { width: Math.max(sizeBefore.width, sizeAfter.width), @@ -80,7 +106,7 @@ class ImageDiff { // the container may be hidden by "viewed" checkbox, so use the parent's width for reference this.diffContainerWidth = Math.max(containerEl.closest('.diff-file-box')!.clientWidth - 300, 100); - const imageInfos = [{ + const imagePair: [ImageInfo, ImageInfo] = [{ path: containerEl.getAttribute('data-path-after'), mime: containerEl.getAttribute('data-mime-after'), images: containerEl.querySelectorAll('img.image-after'), // matches 3 @@ -92,7 +118,8 @@ class ImageDiff { boundsInfo: containerEl.querySelector('.bounds-info-before'), }]; - await Promise.all(imageInfos.map(async (info) => { + const svgBoundsInfo: SvgBoundsInfo = {before: null, after: null}; + await Promise.all(imagePair.map(async (info, index) => { const [success] = await Promise.all(Array.from(info.images, (img) => { return loadElem(img, info.path!); })); @@ -102,115 +129,112 @@ class ImageDiff { const resp = await GET(info.path!); const text = await resp.text(); const bounds = getDefaultSvgBoundsIfUndefined(text, info.path!); + svgBoundsInfo[index === 0 ? 'after' : 'before'] = bounds; if (bounds) { - for (const el of info.images) { - el.setAttribute('width', String(bounds.width)); - el.setAttribute('height', String(bounds.height)); - } hideElem(info.boundsInfo!); } } })); - const imagesAfter = imageInfos[0].images; - const imagesBefore = imageInfos[1].images; + const imagesAfter = imagePair[0].images; + const imagesBefore = imagePair[1].images; - this.initSideBySide(createContext(imagesAfter[0], imagesBefore[0])); + this.initSideBySide(createContext(imagesAfter[0], imagesBefore[0], svgBoundsInfo)); if (imagesAfter.length > 0 && imagesBefore.length > 0) { - this.initSwipe(createContext(imagesAfter[1], imagesBefore[1])); - this.initOverlay(createContext(imagesAfter[2], imagesBefore[2])); + this.initSwipe(createContext(imagesAfter[1], imagesBefore[1], svgBoundsInfo)); + this.initOverlay(createContext(imagesAfter[2], imagesBefore[2], svgBoundsInfo)); } queryElemChildren(containerEl, '.image-diff-tabs', (el) => el.classList.remove('is-loading')); } - initSideBySide(sizes: Record) { + initSideBySide(ctx: ImageContext) { let factor = 1; - if (sizes.maxSize.width > (this.diffContainerWidth - 24) / 2) { - factor = (this.diffContainerWidth - 24) / 2 / sizes.maxSize.width; + if (ctx.maxSize.width > (this.diffContainerWidth - 24) / 2) { + factor = (this.diffContainerWidth - 24) / 2 / ctx.maxSize.width; } - const widthChanged = sizes.imageAfter && sizes.imageBefore && sizes.imageAfter.naturalWidth !== sizes.imageBefore.naturalWidth; - const heightChanged = sizes.imageAfter && sizes.imageBefore && sizes.imageAfter.naturalHeight !== sizes.imageBefore.naturalHeight; - if (sizes.imageAfter) { + const widthChanged = ctx.imageAfter && ctx.imageBefore && ctx.imageAfter.naturalWidth !== ctx.imageBefore.naturalWidth; + const heightChanged = ctx.imageAfter && ctx.imageBefore && ctx.imageAfter.naturalHeight !== ctx.imageBefore.naturalHeight; + if (ctx.imageAfter) { const boundsInfoAfterWidth = this.containerEl.querySelector('.bounds-info-after .bounds-info-width'); if (boundsInfoAfterWidth) { - boundsInfoAfterWidth.textContent = `${sizes.imageAfter.naturalWidth}px`; + boundsInfoAfterWidth.textContent = `${ctx.imageAfter.naturalWidth}px`; boundsInfoAfterWidth.classList.toggle('green', widthChanged); } const boundsInfoAfterHeight = this.containerEl.querySelector('.bounds-info-after .bounds-info-height'); if (boundsInfoAfterHeight) { - boundsInfoAfterHeight.textContent = `${sizes.imageAfter.naturalHeight}px`; + boundsInfoAfterHeight.textContent = `${ctx.imageAfter.naturalHeight}px`; boundsInfoAfterHeight.classList.toggle('green', heightChanged); } } - if (sizes.imageBefore) { + if (ctx.imageBefore) { const boundsInfoBeforeWidth = this.containerEl.querySelector('.bounds-info-before .bounds-info-width'); if (boundsInfoBeforeWidth) { - boundsInfoBeforeWidth.textContent = `${sizes.imageBefore.naturalWidth}px`; + boundsInfoBeforeWidth.textContent = `${ctx.imageBefore.naturalWidth}px`; boundsInfoBeforeWidth.classList.toggle('red', widthChanged); } const boundsInfoBeforeHeight = this.containerEl.querySelector('.bounds-info-before .bounds-info-height'); if (boundsInfoBeforeHeight) { - boundsInfoBeforeHeight.textContent = `${sizes.imageBefore.naturalHeight}px`; + boundsInfoBeforeHeight.textContent = `${ctx.imageBefore.naturalHeight}px`; boundsInfoBeforeHeight.classList.toggle('red', heightChanged); } } - if (sizes.imageAfter) { - const container = sizes.imageAfter.parentNode; - sizes.imageAfter.style.width = `${sizes.sizeAfter.width * factor}px`; - sizes.imageAfter.style.height = `${sizes.sizeAfter.height * factor}px`; + if (ctx.imageAfter) { + const container = ctx.imageAfter.parentNode as HTMLElement; + ctx.imageAfter.style.width = `${ctx.sizeAfter.width * factor}px`; + ctx.imageAfter.style.height = `${ctx.sizeAfter.height * factor}px`; container.style.margin = '10px auto'; - container.style.width = `${sizes.sizeAfter.width * factor + 2}px`; - container.style.height = `${sizes.sizeAfter.height * factor + 2}px`; + container.style.width = `${ctx.sizeAfter.width * factor + 2}px`; + container.style.height = `${ctx.sizeAfter.height * factor + 2}px`; } - if (sizes.imageBefore) { - const container = sizes.imageBefore.parentNode; - sizes.imageBefore.style.width = `${sizes.sizeBefore.width * factor}px`; - sizes.imageBefore.style.height = `${sizes.sizeBefore.height * factor}px`; + if (ctx.imageBefore) { + const container = ctx.imageBefore.parentNode as HTMLElement; + ctx.imageBefore.style.width = `${ctx.sizeBefore.width * factor}px`; + ctx.imageBefore.style.height = `${ctx.sizeBefore.height * factor}px`; container.style.margin = '10px auto'; - container.style.width = `${sizes.sizeBefore.width * factor + 2}px`; - container.style.height = `${sizes.sizeBefore.height * factor + 2}px`; + container.style.width = `${ctx.sizeBefore.width * factor + 2}px`; + container.style.height = `${ctx.sizeBefore.height * factor + 2}px`; } } - initSwipe(sizes: Record) { + initSwipe(ctx: ImageContext) { let factor = 1; - if (sizes.maxSize.width > this.diffContainerWidth - 12) { - factor = (this.diffContainerWidth - 12) / sizes.maxSize.width; + if (ctx.maxSize.width > this.diffContainerWidth - 12) { + factor = (this.diffContainerWidth - 12) / ctx.maxSize.width; } - if (sizes.imageAfter) { - const imgParent = sizes.imageAfter.parentNode; - const swipeFrame = imgParent.parentNode; - sizes.imageAfter.style.width = `${sizes.sizeAfter.width * factor}px`; - sizes.imageAfter.style.height = `${sizes.sizeAfter.height * factor}px`; - imgParent.style.margin = `0px ${sizes.ratio[0] * factor}px`; - imgParent.style.width = `${sizes.sizeAfter.width * factor + 2}px`; - imgParent.style.height = `${sizes.sizeAfter.height * factor + 2}px`; - swipeFrame.style.padding = `${sizes.ratio[1] * factor}px 0 0 0`; - swipeFrame.style.width = `${sizes.maxSize.width * factor + 2}px`; + if (ctx.imageAfter) { + const imgParent = ctx.imageAfter.parentNode as HTMLElement; + const swipeFrame = imgParent.parentNode as HTMLElement; + ctx.imageAfter.style.width = `${ctx.sizeAfter.width * factor}px`; + ctx.imageAfter.style.height = `${ctx.sizeAfter.height * factor}px`; + imgParent.style.margin = `0px ${ctx.ratio[0] * factor}px`; + imgParent.style.width = `${ctx.sizeAfter.width * factor + 2}px`; + imgParent.style.height = `${ctx.sizeAfter.height * factor + 2}px`; + swipeFrame.style.padding = `${ctx.ratio[1] * factor}px 0 0 0`; + swipeFrame.style.width = `${ctx.maxSize.width * factor + 2}px`; } - if (sizes.imageBefore) { - const imgParent = sizes.imageBefore.parentNode; - const swipeFrame = imgParent.parentNode; - sizes.imageBefore.style.width = `${sizes.sizeBefore.width * factor}px`; - sizes.imageBefore.style.height = `${sizes.sizeBefore.height * factor}px`; - imgParent.style.margin = `${sizes.ratio[3] * factor}px ${sizes.ratio[2] * factor}px`; - imgParent.style.width = `${sizes.sizeBefore.width * factor + 2}px`; - imgParent.style.height = `${sizes.sizeBefore.height * factor + 2}px`; - swipeFrame.style.width = `${sizes.maxSize.width * factor + 2}px`; - swipeFrame.style.height = `${sizes.maxSize.height * factor + 2}px`; + if (ctx.imageBefore) { + const imgParent = ctx.imageBefore.parentNode as HTMLElement; + const swipeFrame = imgParent.parentNode as HTMLElement; + ctx.imageBefore.style.width = `${ctx.sizeBefore.width * factor}px`; + ctx.imageBefore.style.height = `${ctx.sizeBefore.height * factor}px`; + imgParent.style.margin = `${ctx.ratio[3] * factor}px ${ctx.ratio[2] * factor}px`; + imgParent.style.width = `${ctx.sizeBefore.width * factor + 2}px`; + imgParent.style.height = `${ctx.sizeBefore.height * factor + 2}px`; + swipeFrame.style.width = `${ctx.maxSize.width * factor + 2}px`; + swipeFrame.style.height = `${ctx.maxSize.height * factor + 2}px`; } // extra height for inner "position: absolute" elements const swipe = this.containerEl.querySelector('.diff-swipe'); if (swipe) { - swipe.style.width = `${sizes.maxSize.width * factor + 2}px`; - swipe.style.height = `${sizes.maxSize.height * factor + 30}px`; + swipe.style.width = `${ctx.maxSize.width * factor + 2}px`; + swipe.style.height = `${ctx.maxSize.height * factor + 30}px`; } this.containerEl.querySelector('.swipe-bar')!.addEventListener('mousedown', (e) => { @@ -237,40 +261,40 @@ class ImageDiff { document.addEventListener('mouseup', removeEventListeners); } - initOverlay(sizes: Record) { + initOverlay(ctx: ImageContext) { let factor = 1; - if (sizes.maxSize.width > this.diffContainerWidth - 12) { - factor = (this.diffContainerWidth - 12) / sizes.maxSize.width; + if (ctx.maxSize.width > this.diffContainerWidth - 12) { + factor = (this.diffContainerWidth - 12) / ctx.maxSize.width; } - if (sizes.imageAfter) { - const container = sizes.imageAfter.parentNode; - sizes.imageAfter.style.width = `${sizes.sizeAfter.width * factor}px`; - sizes.imageAfter.style.height = `${sizes.sizeAfter.height * factor}px`; - container.style.margin = `${sizes.ratio[1] * factor}px ${sizes.ratio[0] * factor}px`; - container.style.width = `${sizes.sizeAfter.width * factor + 2}px`; - container.style.height = `${sizes.sizeAfter.height * factor + 2}px`; + if (ctx.imageAfter) { + const container = ctx.imageAfter.parentNode as HTMLElement; + ctx.imageAfter.style.width = `${ctx.sizeAfter.width * factor}px`; + ctx.imageAfter.style.height = `${ctx.sizeAfter.height * factor}px`; + container.style.margin = `${ctx.ratio[1] * factor}px ${ctx.ratio[0] * factor}px`; + container.style.width = `${ctx.sizeAfter.width * factor + 2}px`; + container.style.height = `${ctx.sizeAfter.height * factor + 2}px`; } - if (sizes.imageBefore) { - const container = sizes.imageBefore.parentNode; - const overlayFrame = container.parentNode; - sizes.imageBefore.style.width = `${sizes.sizeBefore.width * factor}px`; - sizes.imageBefore.style.height = `${sizes.sizeBefore.height * factor}px`; - container.style.margin = `${sizes.ratio[3] * factor}px ${sizes.ratio[2] * factor}px`; - container.style.width = `${sizes.sizeBefore.width * factor + 2}px`; - container.style.height = `${sizes.sizeBefore.height * factor + 2}px`; + if (ctx.imageBefore) { + const container = ctx.imageBefore.parentNode as HTMLElement; + const overlayFrame = container.parentNode as HTMLElement; + ctx.imageBefore.style.width = `${ctx.sizeBefore.width * factor}px`; + ctx.imageBefore.style.height = `${ctx.sizeBefore.height * factor}px`; + container.style.margin = `${ctx.ratio[3] * factor}px ${ctx.ratio[2] * factor}px`; + container.style.width = `${ctx.sizeBefore.width * factor + 2}px`; + container.style.height = `${ctx.sizeBefore.height * factor + 2}px`; // some inner elements are `position: absolute`, so the container's height must be large enough - overlayFrame.style.width = `${sizes.maxSize.width * factor + 2}px`; - overlayFrame.style.height = `${sizes.maxSize.height * factor + 2}px`; + overlayFrame.style.width = `${ctx.maxSize.width * factor + 2}px`; + overlayFrame.style.height = `${ctx.maxSize.height * factor + 2}px`; } const rangeInput = this.containerEl.querySelector('input[type="range"]')!; function updateOpacity() { - if (sizes.imageAfter) { - sizes.imageAfter.parentNode.style.opacity = `${Number(rangeInput.value) / 100}`; + if (ctx.imageAfter) { + (ctx.imageAfter.parentNode as HTMLElement).style.opacity = `${Number(rangeInput.value) / 100}`; } } diff --git a/web_src/js/globals.d.ts b/web_src/js/globals.d.ts index 00f1744a95..49ce63d688 100644 --- a/web_src/js/globals.d.ts +++ b/web_src/js/globals.d.ts @@ -12,8 +12,8 @@ declare module '*.vue' { import type {DefineComponent} from 'vue'; const component: DefineComponent; export default component; - // List of named exports from vue components, used to make `tsc` output clean. - // To actually lint .vue files, `vue-tsc` is used because `tsc` can not parse them. + // Here we declare all exports from vue files so `tsc` or `tsgo` can work for + // non-vue files. To lint .vue files, `vue-tsc` must be used. export function initDashboardRepoList(): void; export function initRepositoryActionView(): void; }