feat(bundle): add zim format support
arcad/edge/pipeline/head There was a failure building this commit
Details
arcad/edge/pipeline/head There was a failure building this commit
Details
This commit is contained in:
parent
1544212ab5
commit
a5c67c29d0
16
go.mod
16
go.mod
|
|
@ -1,13 +1,15 @@
|
|||
module forge.cadoles.com/arcad/edge
|
||||
|
||||
go 1.19
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/getsentry/sentry-go v0.25.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.6
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hashicorp/mdns v1.0.5
|
||||
github.com/keegancsmith/rpc v1.3.0
|
||||
github.com/klauspost/compress v1.16.6
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.8
|
||||
github.com/ulikunitz/xz v0.5.11
|
||||
modernc.org/sqlite v1.20.4
|
||||
)
|
||||
|
||||
|
|
@ -62,12 +64,12 @@ require (
|
|||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648
|
||||
go.opencensus.io v0.22.5 // indirect
|
||||
golang.org/x/crypto v0.7.0
|
||||
golang.org/x/crypto v0.10.0
|
||||
golang.org/x/mod v0.10.0
|
||||
golang.org/x/net v0.9.0 // indirect
|
||||
golang.org/x/sys v0.7.0 // indirect
|
||||
golang.org/x/term v0.7.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
golang.org/x/net v0.11.0
|
||||
golang.org/x/sys v0.9.0 // indirect
|
||||
golang.org/x/term v0.9.0 // indirect
|
||||
golang.org/x/text v0.10.0 // indirect
|
||||
golang.org/x/tools v0.8.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
|
|
|
|||
35
go.sum
35
go.sum
|
|
@ -107,6 +107,7 @@ github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxm
|
|||
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
|
||||
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
|
|
@ -178,6 +179,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
|
|||
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
|
|
@ -194,8 +196,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
|
|||
github.com/hashicorp/go.net v0.0.0-20151006203346-104dcad90073/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/mdns v0.0.0-20151206042412-9d85cf22f9f8/go.mod h1:aa76Av3qgPeIQp9Y3qIkTBPieQYNkQ13Kxe7pze9Wb0=
|
||||
github.com/hashicorp/mdns v1.0.5 h1:1M5hW1cunYeoXOqHwEb/GBDDHAFo0Yqb/uz/beC6LbE=
|
||||
|
|
@ -212,6 +214,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
|
|||
github.com/keegancsmith/rpc v1.3.0 h1:wGWOpjcNrZaY8GDYZJfvyxmlLljm3YQWF+p918DXtDk=
|
||||
github.com/keegancsmith/rpc v1.3.0/go.mod h1:6O2xnOGjPyvIPbvp0MdrOe5r6cu1GZ4JoTzpzDhWeo0=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
|
||||
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
|
|
@ -245,6 +249,7 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
|
|||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
|
||||
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||
github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw=
|
||||
|
|
@ -260,6 +265,7 @@ github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CF
|
|||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -293,6 +299,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
|||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
|
||||
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0=
|
||||
github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
|
||||
|
|
@ -322,8 +330,9 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
|
||||
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -398,8 +407,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
|||
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
|
||||
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -464,15 +473,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
|
||||
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
|
||||
golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
@ -484,8 +493,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
|||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
|
||||
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -662,7 +671,9 @@ modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
|
|||
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
|
||||
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
|
||||
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
|
||||
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
|
||||
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
|
||||
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
|
||||
modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
|
||||
modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
|
||||
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||
|
|
@ -676,9 +687,11 @@ modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A
|
|||
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
|
||||
modernc.org/tcl v1.15.0/go.mod h1:xRoGotBZ6dU+Zo2tca+2EqVEeMmOUBzHnhIwq4YrVnE=
|
||||
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
|
||||
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
|
||||
modernc.org/z v1.7.0/go.mod h1:hVdgNMh8ggTuRG1rGU8x+xGRFfiQUIAw0ZqlPy8+HyQ=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
|
|
|||
|
|
@ -17,5 +17,5 @@ misc/client-sdk-testsuite/src/**/*
|
|||
}
|
||||
|
||||
**/*.go {
|
||||
prep: make GOTEST_ARGS="-short" test
|
||||
# prep: make GOTEST_ARGS="-short" test
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
ModuleFactories []ServerModuleFactory
|
||||
ErrorHandler func(ctx context.Context, err error)
|
||||
}
|
||||
|
||||
type OptionFunc func(opts *Options)
|
||||
|
||||
func NewOptions(funcs ...OptionFunc) *Options {
|
||||
opts := &Options{
|
||||
ModuleFactories: make([]ServerModuleFactory, 0),
|
||||
ErrorHandler: func(ctx context.Context, err error) {
|
||||
logger.Error(ctx, err.Error(), logger.E(errors.WithStack(err)))
|
||||
},
|
||||
}
|
||||
|
||||
for _, fn := range funcs {
|
||||
fn(opts)
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
func WithModulesFactories(factories ...ServerModuleFactory) OptionFunc {
|
||||
return func(opts *Options) {
|
||||
opts.ModuleFactories = factories
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ type ArchiveExt string
|
|||
const (
|
||||
ExtZip ArchiveExt = "zip"
|
||||
ExtTarGz ArchiveExt = "tar.gz"
|
||||
ExtZim ArchiveExt = "zim"
|
||||
)
|
||||
|
||||
func FromPath(path string) (Bundle, error) {
|
||||
|
|
@ -56,5 +57,14 @@ func matchArchivePattern(archivePath string) (Bundle, error) {
|
|||
return NewZipBundle(archivePath), nil
|
||||
}
|
||||
|
||||
matches, err = filepath.Match(fmt.Sprintf("*.%s", ExtZim), base)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not match file archive '%s'", archivePath)
|
||||
}
|
||||
|
||||
if matches {
|
||||
return NewZimBundle(archivePath), nil
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrUnknownBundleArchiveExt)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
package zim
|
||||
|
||||
import "io"
|
||||
|
||||
type BlobReader interface {
|
||||
io.ReadCloser
|
||||
Size() (int64, error)
|
||||
}
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type CompressedBlobReader struct {
|
||||
reader *Reader
|
||||
decoderFactory BlobDecoderFactory
|
||||
|
||||
clusterStartOffset uint64
|
||||
clusterEndOffset uint64
|
||||
blobIndex uint32
|
||||
blobSize int
|
||||
readOffset uint64
|
||||
|
||||
loadCluster sync.Once
|
||||
loadClusterErr error
|
||||
|
||||
data []byte
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Size implements BlobReader.
|
||||
func (r *CompressedBlobReader) Size() (int64, error) {
|
||||
if err := r.loadClusterData(); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return int64(len(r.data)), nil
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *CompressedBlobReader) Close() error {
|
||||
clear(r.data)
|
||||
r.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *CompressedBlobReader) Read(p []byte) (int, error) {
|
||||
if err := r.loadClusterData(); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
length := len(p)
|
||||
remaining := len(r.data) - int(r.readOffset)
|
||||
if length > remaining {
|
||||
length = remaining
|
||||
}
|
||||
|
||||
chunk := make([]byte, length)
|
||||
|
||||
copy(chunk, r.data[r.readOffset:int(r.readOffset)+length])
|
||||
copy(p, chunk)
|
||||
|
||||
if length == remaining {
|
||||
return length, io.EOF
|
||||
}
|
||||
|
||||
r.readOffset += uint64(length)
|
||||
|
||||
return length, nil
|
||||
}
|
||||
|
||||
func (r *CompressedBlobReader) loadClusterData() error {
|
||||
if r.closed {
|
||||
return errors.WithStack(os.ErrClosed)
|
||||
}
|
||||
|
||||
r.loadCluster.Do(func() {
|
||||
compressedData := make([]byte, r.clusterEndOffset-r.clusterStartOffset)
|
||||
if err := r.reader.readRange(int64(r.clusterStartOffset+1), compressedData); err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobBuffer := bytes.NewBuffer(compressedData)
|
||||
|
||||
decoder, err := r.decoderFactory(blobBuffer)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer decoder.Close()
|
||||
|
||||
uncompressedData, err := io.ReadAll(decoder)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
blobStart uint64
|
||||
blobEnd uint64
|
||||
)
|
||||
|
||||
if r.blobSize == 8 {
|
||||
blobStart64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobStart = blobStart64
|
||||
|
||||
blobEnd64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobEnd = blobEnd64
|
||||
} else {
|
||||
blobStart32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobStart = uint64(blobStart32)
|
||||
|
||||
blobEnd32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobEnd = uint64(blobEnd32)
|
||||
}
|
||||
|
||||
r.data = make([]byte, blobEnd-blobStart)
|
||||
copy(r.data, uncompressedData[blobStart:blobEnd])
|
||||
})
|
||||
if r.loadClusterErr != nil {
|
||||
return errors.WithStack(r.loadClusterErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type BlobDecoderFactory func(io.Reader) (io.ReadCloser, error)
|
||||
|
||||
func NewCompressedBlobReader(reader *Reader, decoderFactory BlobDecoderFactory, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
|
||||
return &CompressedBlobReader{
|
||||
reader: reader,
|
||||
decoderFactory: decoderFactory,
|
||||
clusterStartOffset: clusterStartOffset,
|
||||
clusterEndOffset: clusterEndOffset,
|
||||
blobIndex: blobIndex,
|
||||
blobSize: blobSize,
|
||||
readOffset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
var _ BlobReader = &UncompressedBlobReader{}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type zimCompression int
|
||||
|
||||
const (
|
||||
zimCompressionNoneZeno zimCompression = 0
|
||||
zimCompressionNone zimCompression = 1
|
||||
zimCompressionNoneZLib zimCompression = 2
|
||||
zimCompressionNoneBZip2 zimCompression = 3
|
||||
zimCompressionNoneXZ zimCompression = 4
|
||||
zimCompressionNoneZStandard zimCompression = 5
|
||||
)
|
||||
|
||||
type ContentEntry struct {
|
||||
*BaseEntry
|
||||
mimeType string
|
||||
clusterIndex uint32
|
||||
blobIndex uint32
|
||||
}
|
||||
|
||||
func (e *ContentEntry) Compression() (int, error) {
|
||||
clusterHeader, _, _, err := e.readClusterInfo()
|
||||
if err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return int((clusterHeader << 4) >> 4), nil
|
||||
}
|
||||
|
||||
func (e *ContentEntry) MimeType() string {
|
||||
return e.mimeType
|
||||
}
|
||||
|
||||
func (e *ContentEntry) Reader() (BlobReader, error) {
|
||||
clusterHeader, clusterStartOffset, clusterEndOffset, err := e.readClusterInfo()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
compression := (clusterHeader << 4) >> 4
|
||||
extended := (clusterHeader<<3)>>7 == 1
|
||||
|
||||
blobSize := 4
|
||||
if extended {
|
||||
blobSize = 8
|
||||
}
|
||||
|
||||
switch compression {
|
||||
|
||||
// Uncompressed blobs
|
||||
case uint8(zimCompressionNoneZeno):
|
||||
fallthrough
|
||||
case uint8(zimCompressionNone):
|
||||
startPos := clusterStartOffset + 1
|
||||
blobOffset := uint64(e.blobIndex * uint32(blobSize))
|
||||
|
||||
data := make([]byte, 2*blobSize)
|
||||
if err := e.reader.readRange(int64(startPos+blobOffset), data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var (
|
||||
blobStart uint64
|
||||
blobEnd uint64
|
||||
)
|
||||
|
||||
if extended {
|
||||
blobStart64, err := readUint64(data[0:blobSize], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobStart = blobStart64
|
||||
|
||||
blobEnd64, err := readUint64(data[blobSize:blobSize*2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobEnd = uint64(blobEnd64)
|
||||
} else {
|
||||
blobStart32, err := readUint32(data[0:blobSize], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobStart = uint64(blobStart32)
|
||||
|
||||
blobEnd32, err := readUint32(data[blobSize:blobSize*2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobEnd = uint64(blobEnd32)
|
||||
}
|
||||
|
||||
return NewUncompressedBlobReader(e.reader, startPos+blobStart, startPos+blobEnd, blobSize), nil
|
||||
|
||||
// Supported compression algorithms
|
||||
case uint8(zimCompressionNoneXZ):
|
||||
return NewXZBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil
|
||||
|
||||
case uint8(zimCompressionNoneZStandard):
|
||||
return NewZStdBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil
|
||||
|
||||
// Unsupported compression algorithms
|
||||
case uint8(zimCompressionNoneZLib):
|
||||
fallthrough
|
||||
case uint8(zimCompressionNoneBZip2):
|
||||
fallthrough
|
||||
default:
|
||||
return nil, errors.Wrapf(ErrCompressionAlgorithmNotSupported, "unexpected compression algorithm '%d'", compression)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ContentEntry) Redirect() (*ContentEntry, error) {
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func (e *ContentEntry) readClusterInfo() (uint8, uint64, uint64, error) {
|
||||
startClusterOffset, clusterEndOffset, err := e.reader.getClusterOffsets(int(e.clusterIndex))
|
||||
if err != nil {
|
||||
return 0, 0, 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
data := make([]byte, 1)
|
||||
if err := e.reader.readRange(int64(startClusterOffset), data); err != nil {
|
||||
return 0, 0, 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
clusterHeader := uint8(data[0])
|
||||
|
||||
return clusterHeader, startClusterOffset, clusterEndOffset, nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseContentEntry(offset int64, base *BaseEntry) (*ContentEntry, error) {
|
||||
entry := &ContentEntry{
|
||||
BaseEntry: base,
|
||||
}
|
||||
|
||||
data := make([]byte, 16)
|
||||
if err := r.readRange(offset, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if mimeTypeIndex >= uint16(len(r.mimeTypes)) {
|
||||
return nil, errors.Errorf("mime type index '%d' greater than mime types length '%d'", mimeTypeIndex, len(r.mimeTypes))
|
||||
}
|
||||
|
||||
entry.mimeType = r.mimeTypes[mimeTypeIndex]
|
||||
|
||||
entry.namespace = Namespace(data[3:4])
|
||||
|
||||
clusterIndex, err := readUint32(data[8:12], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.clusterIndex = clusterIndex
|
||||
|
||||
blobIndex, err := readUint32(data[12:16], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.blobIndex = blobIndex
|
||||
|
||||
strs, _, err := r.readStringsAt(offset+16, 2, 1024)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if len(strs) > 0 {
|
||||
entry.url = strs[0]
|
||||
}
|
||||
|
||||
if len(strs) > 1 {
|
||||
entry.title = strs[1]
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type Entry interface {
|
||||
Redirect() (*ContentEntry, error)
|
||||
Namespace() Namespace
|
||||
URL() string
|
||||
FullURL() string
|
||||
Title() string
|
||||
}
|
||||
|
||||
type BaseEntry struct {
|
||||
mimeTypeIndex uint16
|
||||
namespace Namespace
|
||||
url string
|
||||
title string
|
||||
reader *Reader
|
||||
}
|
||||
|
||||
func (e *BaseEntry) Namespace() Namespace {
|
||||
return e.namespace
|
||||
}
|
||||
|
||||
func (e *BaseEntry) Title() string {
|
||||
if e.title == "" {
|
||||
return e.url
|
||||
}
|
||||
|
||||
return e.title
|
||||
}
|
||||
|
||||
func (e *BaseEntry) URL() string {
|
||||
return e.url
|
||||
}
|
||||
|
||||
func (e *BaseEntry) FullURL() string {
|
||||
return toFullURL(e.Namespace(), e.URL())
|
||||
}
|
||||
|
||||
func (r *Reader) parseBaseEntry(offset int64) (*BaseEntry, error) {
|
||||
entry := &BaseEntry{
|
||||
reader: r,
|
||||
}
|
||||
|
||||
data := make([]byte, 3)
|
||||
if err := r.readRange(offset, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.mimeTypeIndex = mimeTypeIndex
|
||||
entry.namespace = Namespace(data[2])
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
type RedirectEntry struct {
|
||||
*BaseEntry
|
||||
redirectIndex uint32
|
||||
}
|
||||
|
||||
func (e *RedirectEntry) Redirect() (*ContentEntry, error) {
|
||||
if e.redirectIndex >= uint32(len(e.reader.urlIndex)) {
|
||||
return nil, errors.Wrapf(ErrInvalidIndex, "entry index '%d' out of bounds", e.redirectIndex)
|
||||
}
|
||||
|
||||
entryPtr := e.reader.urlIndex[e.redirectIndex]
|
||||
entry, err := e.reader.parseEntryAt(int64(entryPtr))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry, err = entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
contentEntry, ok := entry.(*ContentEntry)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(ErrInvalidRedirect)
|
||||
}
|
||||
|
||||
return contentEntry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseRedirectEntry(offset int64, base *BaseEntry) (*RedirectEntry, error) {
|
||||
entry := &RedirectEntry{
|
||||
BaseEntry: base,
|
||||
}
|
||||
|
||||
data := make([]byte, 4)
|
||||
if err := r.readRange(offset+8, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
redirectIndex, err := readUint32(data, binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.redirectIndex = redirectIndex
|
||||
|
||||
strs, _, err := r.readStringsAt(offset+12, 2, 1024)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if len(strs) > 0 {
|
||||
entry.url = strs[0]
|
||||
}
|
||||
|
||||
if len(strs) > 1 {
|
||||
entry.title = strs[1]
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func toFullURL(ns Namespace, url string) string {
|
||||
if ns == "\x00" {
|
||||
return url
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s", ns, url)
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package zim
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
type EntryIterator struct {
|
||||
index int
|
||||
entry Entry
|
||||
err error
|
||||
reader *Reader
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Next() bool {
|
||||
if it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
entryCount := it.reader.EntryCount()
|
||||
|
||||
if it.index >= int(entryCount-1) {
|
||||
return false
|
||||
}
|
||||
|
||||
entry, err := it.reader.EntryAt(it.index)
|
||||
if err != nil {
|
||||
it.err = errors.WithStack(err)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
it.entry = entry
|
||||
it.index++
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Index() int {
|
||||
return it.index - 1
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Entry() Entry {
|
||||
return it.entry
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package zim
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrInvalidIndex = errors.New("invalid index")
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrInvalidRedirect = errors.New("invalid redirect")
|
||||
ErrCompressionAlgorithmNotSupported = errors.New("compression algorithm not supported")
|
||||
)
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
package zim
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
func (r *Reader) Favicon() (*ContentEntry, error) {
|
||||
illustration, err := r.getMetadataIllustration()
|
||||
if err != nil && !errors.Is(err, ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if illustration != nil {
|
||||
return illustration, nil
|
||||
}
|
||||
|
||||
namespaces := []Namespace{V5NamespaceLayout, V5NamespaceImageFile}
|
||||
urls := []string{"favicon", "favicon.png"}
|
||||
|
||||
for _, ns := range namespaces {
|
||||
for _, url := range urls {
|
||||
entry, err := r.EntryWithURL(ns, url)
|
||||
if err != nil && !errors.Is(err, ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return content, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
func (r *Reader) getMetadataIllustration() (*ContentEntry, error) {
|
||||
keys := []MetadataKey{MetadataIllustration96x96at2, MetadataIllustration48x48at1}
|
||||
|
||||
metadata, err := r.Metadata(keys...)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
for _, k := range keys {
|
||||
if _, exists := metadata[k]; exists {
|
||||
entry, err := r.EntryWithURL(V5NamespaceMetadata, string(k))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return content, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type MetadataKey string
|
||||
|
||||
// See https://wiki.openzim.org/wiki/Metadata
|
||||
const (
|
||||
MetadataName MetadataKey = "Name"
|
||||
MetadataTitle MetadataKey = "Title"
|
||||
MetadataDescription MetadataKey = "Description"
|
||||
MetadataLongDescription MetadataKey = "LongDescription"
|
||||
MetadataCreator MetadataKey = "Creator"
|
||||
MetadataTags MetadataKey = "Tags"
|
||||
MetadataDate MetadataKey = "Date"
|
||||
MetadataPublisher MetadataKey = "Publisher"
|
||||
MetadataFlavour MetadataKey = "Flavour"
|
||||
MetadataSource MetadataKey = "Source"
|
||||
MetadataLanguage MetadataKey = "Language"
|
||||
MetadataIllustration48x48at1 MetadataKey = "Illustration_48x48@1"
|
||||
MetadataIllustration96x96at2 MetadataKey = "Illustration_96x96@2"
|
||||
)
|
||||
|
||||
var knownKeys = []MetadataKey{
|
||||
MetadataName,
|
||||
MetadataTitle,
|
||||
MetadataDescription,
|
||||
MetadataLongDescription,
|
||||
MetadataCreator,
|
||||
MetadataPublisher,
|
||||
MetadataLanguage,
|
||||
MetadataTags,
|
||||
MetadataDate,
|
||||
MetadataFlavour,
|
||||
MetadataSource,
|
||||
MetadataIllustration48x48at1,
|
||||
MetadataIllustration96x96at2,
|
||||
}
|
||||
|
||||
// Metadata returns a copy of the internal metadata map of the ZIM file.
|
||||
func (r *Reader) Metadata(keys ...MetadataKey) (map[MetadataKey]string, error) {
|
||||
if len(keys) == 0 {
|
||||
keys = knownKeys
|
||||
}
|
||||
|
||||
metadata := make(map[MetadataKey]string)
|
||||
|
||||
for _, key := range keys {
|
||||
entry, err := r.EntryWithURL(V5NamespaceMetadata, string(key))
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
reader, err := content.Reader()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
metadata[key] = string(data)
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
package zim
|
||||
|
||||
type Namespace string
|
||||
|
||||
const (
|
||||
V6NamespaceContent Namespace = "C"
|
||||
V6NamespaceMetadata Namespace = "M"
|
||||
V6NamespaceWellKnown Namespace = "W"
|
||||
V6NamespaceSearch Namespace = "X"
|
||||
)
|
||||
|
||||
const (
|
||||
V5NamespaceLayout Namespace = "-"
|
||||
V5NamespaceArticle Namespace = "A"
|
||||
V5NamespaceArticleMetadata Namespace = "B"
|
||||
V5NamespaceImageFile Namespace = "I"
|
||||
V5NamespaceImageText Namespace = "J"
|
||||
V5NamespaceMetadata Namespace = "M"
|
||||
V5NamespaceCategoryText Namespace = "U"
|
||||
V5NamespaceCategoryArticleList Namespace = "V"
|
||||
V5NamespaceCategoryPerArticle Namespace = "W"
|
||||
V5NamespaceSearch Namespace = "X"
|
||||
)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
package zim
|
||||
|
||||
import "time"
|
||||
|
||||
type Options struct {
|
||||
URLCacheSize int
|
||||
URLCacheTTL time.Duration
|
||||
CacheSize int
|
||||
}
|
||||
|
||||
type OptionFunc func(opts *Options)
|
||||
|
||||
func NewOptions(funcs ...OptionFunc) *Options {
|
||||
funcs = append([]OptionFunc{
|
||||
WithCacheSize(2048),
|
||||
}, funcs...)
|
||||
|
||||
opts := &Options{}
|
||||
for _, fn := range funcs {
|
||||
fn(opts)
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
func WithCacheSize(size int) OptionFunc {
|
||||
return func(opts *Options) {
|
||||
opts.CacheSize = size
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,558 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
const zimFormatMagicNumber uint32 = 0x44D495A
|
||||
const nullByte = '\x00'
|
||||
const zimRedirect = 0xffff
|
||||
|
||||
type Reader struct {
|
||||
majorVersion uint16
|
||||
minorVersion uint16
|
||||
uuid string
|
||||
entryCount uint32
|
||||
clusterCount uint32
|
||||
urlPtrPos uint64
|
||||
titlePtrPos uint64
|
||||
clusterPtrPos uint64
|
||||
mimeListPos uint64
|
||||
mainPage uint32
|
||||
layoutPage uint32
|
||||
checksumPos uint64
|
||||
|
||||
mimeTypes []string
|
||||
urlIndex []uint64
|
||||
clusterIndex []uint64
|
||||
|
||||
cache *lru.Cache[string, Entry]
|
||||
urls map[string]int
|
||||
|
||||
rangeReader RangeReadCloser
|
||||
}
|
||||
|
||||
func (r *Reader) Version() (majorVersion, minorVersion uint16) {
|
||||
return r.majorVersion, r.minorVersion
|
||||
}
|
||||
|
||||
func (r *Reader) EntryCount() uint32 {
|
||||
return r.entryCount
|
||||
}
|
||||
|
||||
func (r *Reader) ClusterCount() uint32 {
|
||||
return r.clusterCount
|
||||
}
|
||||
|
||||
func (r *Reader) UUID() string {
|
||||
return r.uuid
|
||||
}
|
||||
|
||||
func (r *Reader) Close() error {
|
||||
if err := r.rangeReader.Close(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) MainPage() (Entry, error) {
|
||||
if r.mainPage == 0xffffffff {
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
entry, err := r.EntryAt(int(r.mainPage))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) Entries() *EntryIterator {
|
||||
return &EntryIterator{
|
||||
reader: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reader) EntryAt(idx int) (Entry, error) {
|
||||
if idx >= len(r.urlIndex) || idx < 0 {
|
||||
return nil, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", idx)
|
||||
}
|
||||
|
||||
entryPtr := r.urlIndex[idx]
|
||||
|
||||
entry, err := r.parseEntryAt(int64(entryPtr))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.cacheEntry(entryPtr, entry)
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) EntryWithFullURL(url string) (Entry, error) {
|
||||
urlNum, exists := r.urls[url]
|
||||
if !exists {
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
entry, err := r.EntryAt(urlNum)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) EntryWithURL(ns Namespace, url string) (Entry, error) {
|
||||
fullURL := toFullURL(ns, url)
|
||||
|
||||
entry, err := r.EntryWithFullURL(fullURL)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) EntryWithTitle(ns Namespace, title string) (Entry, error) {
|
||||
entry, found := r.getEntryByTitleFromCache(ns, title)
|
||||
if found {
|
||||
logger.Debug(context.Background(), "found entry with title from cache", logger.F("entry", entry.FullURL()))
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
iterator := r.Entries()
|
||||
|
||||
for iterator.Next() {
|
||||
entry := iterator.Entry()
|
||||
|
||||
if entry.Title() == title && entry.Namespace() == ns {
|
||||
return entry, nil
|
||||
}
|
||||
}
|
||||
if err := iterator.Err(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
func (r *Reader) getURLCacheKey(fullURL string) string {
|
||||
return "url:" + fullURL
|
||||
}
|
||||
|
||||
func (r *Reader) getTitleCacheKey(ns Namespace, title string) string {
|
||||
return fmt.Sprintf("title:%s/%s", ns, title)
|
||||
}
|
||||
|
||||
func (r *Reader) cacheEntry(offset uint64, entry Entry) {
|
||||
urlKey := r.getURLCacheKey(entry.FullURL())
|
||||
titleKey := r.getTitleCacheKey(entry.Namespace(), entry.Title())
|
||||
|
||||
_, urlFound := r.cache.Peek(urlKey)
|
||||
_, titleFound := r.cache.Peek(titleKey)
|
||||
|
||||
if urlFound && titleFound {
|
||||
return
|
||||
}
|
||||
|
||||
r.cache.Add(urlKey, entry)
|
||||
r.cache.Add(titleKey, entry)
|
||||
}
|
||||
|
||||
func (r *Reader) getEntryByTitleFromCache(namespace Namespace, title string) (Entry, bool) {
|
||||
key := r.getTitleCacheKey(namespace, title)
|
||||
return r.cache.Get(key)
|
||||
}
|
||||
|
||||
func (r *Reader) parse() error {
|
||||
if err := r.parseHeader(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := r.parseMimeTypes(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := r.parseURLIndex(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := r.parseClusterIndex(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseHeader() error {
|
||||
header := make([]byte, 80)
|
||||
if err := r.readRange(0, header); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
magicNumber, err := readUint32(header[0:4], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if magicNumber != zimFormatMagicNumber {
|
||||
return errors.Errorf("invalid zim magic number '%d'", magicNumber)
|
||||
}
|
||||
|
||||
majorVersion, err := readUint16(header[4:6], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.majorVersion = majorVersion
|
||||
|
||||
minorVersion, err := readUint16(header[6:8], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.minorVersion = minorVersion
|
||||
|
||||
if err := r.parseUUID(header[8:16]); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
entryCount, err := readUint32(header[24:28], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.entryCount = entryCount
|
||||
|
||||
clusterCount, err := readUint32(header[28:32], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.clusterCount = clusterCount
|
||||
|
||||
urlPtrPos, err := readUint64(header[32:40], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.urlPtrPos = urlPtrPos
|
||||
|
||||
titlePtrPos, err := readUint64(header[40:48], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.titlePtrPos = titlePtrPos
|
||||
|
||||
clusterPtrPos, err := readUint64(header[48:56], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.clusterPtrPos = clusterPtrPos
|
||||
|
||||
mimeListPos, err := readUint64(header[56:64], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.mimeListPos = mimeListPos
|
||||
|
||||
mainPage, err := readUint32(header[64:68], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.mainPage = mainPage
|
||||
|
||||
layoutPage, err := readUint32(header[68:72], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.layoutPage = layoutPage
|
||||
|
||||
checksumPos, err := readUint64(header[72:80], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.checksumPos = checksumPos
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseUUID(data []byte) error {
|
||||
parts := make([]string, 0, 5)
|
||||
|
||||
val32, err := readUint32(data[0:4], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%08x", val32))
|
||||
|
||||
val16, err := readUint16(data[4:6], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%04x", val16))
|
||||
|
||||
val16, err = readUint16(data[6:8], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%04x", val16))
|
||||
|
||||
val16, err = readUint16(data[8:10], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%04x", val16))
|
||||
|
||||
val32, err = readUint32(data[10:14], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
val16, err = readUint16(data[14:16], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%x%x", val32, val16))
|
||||
|
||||
r.uuid = strings.Join(parts, "-")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseMimeTypes() error {
|
||||
mimeTypes := make([]string, 0)
|
||||
offset := int64(r.mimeListPos)
|
||||
read := int64(0)
|
||||
var err error
|
||||
var found []string
|
||||
for {
|
||||
found, read, err = r.readStringsAt(offset+read, 64, 1024)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if len(found) == 0 || found[0] == "" {
|
||||
break
|
||||
}
|
||||
|
||||
mimeTypes = append(mimeTypes, found...)
|
||||
}
|
||||
|
||||
r.mimeTypes = mimeTypes
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseURLIndex() error {
|
||||
urlIndex, err := r.parsePointerIndex(int64(r.urlPtrPos), int64(r.entryCount))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.urlIndex = urlIndex
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseClusterIndex() error {
|
||||
clusterIndex, err := r.parsePointerIndex(int64(r.clusterPtrPos), int64(r.clusterCount+1))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.clusterIndex = clusterIndex
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseEntryAt(offset int64) (Entry, error) {
|
||||
base, err := r.parseBaseEntry(offset)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var entry Entry
|
||||
|
||||
if base.mimeTypeIndex == zimRedirect {
|
||||
entry, err = r.parseRedirectEntry(offset, base)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
} else {
|
||||
entry, err = r.parseContentEntry(offset, base)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) parsePointerIndex(startAddr int64, count int64) ([]uint64, error) {
|
||||
index := make([]uint64, count)
|
||||
|
||||
data := make([]byte, count*8)
|
||||
if err := r.readRange(startAddr, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
for i := int64(0); i < count; i++ {
|
||||
offset := i * 8
|
||||
ptr, err := readUint64(data[offset:offset+8], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
index[i] = ptr
|
||||
}
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (r *Reader) getClusterOffsets(clusterNum int) (uint64, uint64, error) {
|
||||
if clusterNum > len(r.clusterIndex)-1 || clusterNum < 0 {
|
||||
return 0, 0, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", clusterNum)
|
||||
}
|
||||
|
||||
return r.clusterIndex[clusterNum], r.clusterIndex[clusterNum+1] - 1, nil
|
||||
}
|
||||
|
||||
func (r *Reader) preload() error {
|
||||
r.urls = make(map[string]int, r.entryCount)
|
||||
|
||||
iterator := r.Entries()
|
||||
for iterator.Next() {
|
||||
entry := iterator.Entry()
|
||||
r.urls[entry.FullURL()] = iterator.Index()
|
||||
}
|
||||
if err := iterator.Err(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) readRange(offset int64, v []byte) error {
|
||||
read, err := r.rangeReader.ReadAt(v, offset)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if read != len(v) {
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) readStringsAt(offset int64, count int, bufferSize int) ([]string, int64, error) {
|
||||
var sb strings.Builder
|
||||
read := int64(0)
|
||||
|
||||
values := make([]string, 0, count)
|
||||
wasNullByte := false
|
||||
|
||||
for {
|
||||
data := make([]byte, bufferSize)
|
||||
err := r.readRange(offset+read, data)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, read, errors.WithStack(err)
|
||||
}
|
||||
|
||||
for idx := 0; idx < len(data); idx++ {
|
||||
d := data[idx]
|
||||
if err := sb.WriteByte(d); err != nil {
|
||||
return nil, read, errors.WithStack(err)
|
||||
}
|
||||
|
||||
read++
|
||||
|
||||
if d == nullByte {
|
||||
if wasNullByte {
|
||||
return values, read, nil
|
||||
}
|
||||
|
||||
wasNullByte = true
|
||||
|
||||
str := strings.TrimRight(sb.String(), "\x00")
|
||||
values = append(values, str)
|
||||
|
||||
if len(values) == count || errors.Is(err, io.EOF) {
|
||||
return values, read, nil
|
||||
}
|
||||
|
||||
sb.Reset()
|
||||
} else {
|
||||
wasNullByte = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type RangeReadCloser interface {
|
||||
io.Closer
|
||||
ReadAt(data []byte, offset int64) (n int, err error)
|
||||
}
|
||||
|
||||
func NewReader(rangeReader RangeReadCloser, funcs ...OptionFunc) (*Reader, error) {
|
||||
opts := NewOptions(funcs...)
|
||||
|
||||
cache, err := lru.New[string, Entry](opts.CacheSize)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
reader := &Reader{
|
||||
rangeReader: rangeReader,
|
||||
cache: cache,
|
||||
}
|
||||
|
||||
if err := reader.parse(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := reader.preload(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func Open(path string, funcs ...OptionFunc) (*Reader, error) {
|
||||
file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
reader, err := NewReader(file, funcs...)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type readerTestCase struct {
|
||||
UUID string `json:"uuid"`
|
||||
EntryCount uint32 `json:"entryCount"`
|
||||
Entries []struct {
|
||||
Namespace Namespace `json:"namespace"`
|
||||
URL string `json:"url"`
|
||||
Size int64 `json:"size"`
|
||||
Compression int `json:"compression"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Title string `json:"title"`
|
||||
} `json:"entries"`
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(logger.LevelDebug)
|
||||
logger.SetFormat(logger.FormatHuman)
|
||||
}
|
||||
|
||||
files, err := filepath.Glob("testdata/*.zim")
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
for _, zf := range files {
|
||||
testName := filepath.Base(zf)
|
||||
testCase, err := loadZimFileTestCase(zf)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
reader, err := Open(zf)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
}()
|
||||
|
||||
if e, g := testCase.UUID, reader.UUID(); e != g {
|
||||
t.Errorf("reader.UUID(): expected '%s', got '%s'", e, g)
|
||||
}
|
||||
|
||||
if e, g := testCase.EntryCount, reader.EntryCount(); e != g {
|
||||
t.Errorf("reader.EntryCount(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
if testCase.Entries == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, entryTestCase := range testCase.Entries {
|
||||
testName := fmt.Sprintf("Entry/%s/%s", entryTestCase.Namespace, entryTestCase.URL)
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
entry, err := reader.EntryWithURL(entryTestCase.Namespace, entryTestCase.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.MimeType, content.MimeType(); e != g {
|
||||
t.Errorf("content.MimeType(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.Title, content.Title(); e != g {
|
||||
t.Errorf("content.Title(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
compression, err := content.Compression()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.Compression, compression; e != g {
|
||||
t.Errorf("content.Compression(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
contentReader, err := content.Reader()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
size, err := contentReader.Size()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.Size, size; e != g {
|
||||
t.Errorf("content.Size(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadZimFileTestCase(zimFile string) (*readerTestCase, error) {
|
||||
testCaseFile, _ := strings.CutSuffix(zimFile, ".zim")
|
||||
|
||||
data, err := os.ReadFile(testCaseFile + ".json")
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
testCase := &readerTestCase{}
|
||||
if err := json.Unmarshal(data, testCase); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return testCase, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"uuid": "8d141c3b-115d-bf73-294a-ee3c2e6b97b0",
|
||||
"entryCount": 6223,
|
||||
"entries": [
|
||||
{
|
||||
"namespace": "C",
|
||||
"url": "users_page=9",
|
||||
"compression": 5,
|
||||
"size": 58646,
|
||||
"mimeType": "text/html",
|
||||
"title": "users_page=9"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,22 @@
|
|||
{
|
||||
"uuid": "cf81f094-d802-c790-b854-c74ad9701ddb",
|
||||
"entryCount": 271,
|
||||
"entries": [
|
||||
{
|
||||
"namespace": "C",
|
||||
"url": "blog/202206-ShowroomInnovation.jpg",
|
||||
"compression": 1,
|
||||
"size": 260260,
|
||||
"mimeType": "image/jpeg",
|
||||
"title": "blog/202206-ShowroomInnovation.jpg"
|
||||
},
|
||||
{
|
||||
"namespace": "C",
|
||||
"url": "team/index.html",
|
||||
"compression": 5,
|
||||
"size": 93185,
|
||||
"mimeType": "text/html",
|
||||
"title": "Cadoles - Notre équipe"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"uuid": "ad4f406c-2021-2db8-c729-297568bbe376",
|
||||
"entryCount": 330,
|
||||
"entries": [
|
||||
{
|
||||
"namespace": "M",
|
||||
"url": "Illustration_48x48@1",
|
||||
"compression": 5,
|
||||
"size": 5365,
|
||||
"mimeType": "text/plain",
|
||||
"title": "Illustration_48x48@1"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,86 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type UncompressedBlobReader struct {
|
||||
reader *Reader
|
||||
blobStartOffset uint64
|
||||
blobEndOffset uint64
|
||||
blobSize int
|
||||
readOffset int
|
||||
|
||||
blobData []byte
|
||||
loadBlobOnce sync.Once
|
||||
loadBlobErr error
|
||||
}
|
||||
|
||||
// Size implements BlobReader.
|
||||
func (r *UncompressedBlobReader) Size() (int64, error) {
|
||||
return int64(r.blobEndOffset - r.blobStartOffset), nil
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *UncompressedBlobReader) Close() error {
|
||||
clear(r.blobData)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *UncompressedBlobReader) Read(p []byte) (n int, err error) {
|
||||
blobData, err := r.loadBlob()
|
||||
if err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
chunkLength := len(p)
|
||||
remaining := int(len(blobData) - r.readOffset)
|
||||
if chunkLength > remaining {
|
||||
chunkLength = remaining
|
||||
}
|
||||
|
||||
chunk := blobData[r.readOffset : r.readOffset+chunkLength]
|
||||
r.readOffset += chunkLength
|
||||
|
||||
copy(p, chunk)
|
||||
|
||||
if chunkLength == remaining {
|
||||
return chunkLength, io.EOF
|
||||
}
|
||||
|
||||
return chunkLength, nil
|
||||
}
|
||||
|
||||
func (r *UncompressedBlobReader) loadBlob() ([]byte, error) {
|
||||
r.loadBlobOnce.Do(func() {
|
||||
data := make([]byte, r.blobEndOffset-r.blobStartOffset)
|
||||
err := r.reader.readRange(int64(r.blobStartOffset), data)
|
||||
if err != nil {
|
||||
r.loadBlobErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
r.blobData = data
|
||||
})
|
||||
if r.loadBlobErr != nil {
|
||||
return nil, errors.WithStack(r.loadBlobErr)
|
||||
}
|
||||
|
||||
return r.blobData, nil
|
||||
}
|
||||
|
||||
func NewUncompressedBlobReader(reader *Reader, blobStartOffset, blobEndOffset uint64, blobSize int) *UncompressedBlobReader {
|
||||
return &UncompressedBlobReader{
|
||||
reader: reader,
|
||||
blobStartOffset: blobStartOffset,
|
||||
blobEndOffset: blobEndOffset,
|
||||
blobSize: blobSize,
|
||||
readOffset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
var _ BlobReader = &UncompressedBlobReader{}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// read a little endian uint64
|
||||
func readUint64(b []byte, order binary.ByteOrder) (uint64, error) {
|
||||
var v uint64
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// read a little endian uint32
|
||||
func readUint32(b []byte, order binary.ByteOrder) (uint32, error) {
|
||||
var v uint32
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// read a little endian uint16
|
||||
func readUint16(b []byte, order binary.ByteOrder) (uint16, error) {
|
||||
var v uint16
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// read a little endian uint8
|
||||
func readUint8(b []byte, order binary.ByteOrder) (uint8, error) {
|
||||
var v uint8
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
type XZBlobReader struct {
|
||||
decoder *xz.Reader
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *XZBlobReader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *XZBlobReader) Read(p []byte) (n int, err error) {
|
||||
return r.decoder.Read(p)
|
||||
}
|
||||
|
||||
var _ io.ReadCloser = &XZBlobReader{}
|
||||
|
||||
func NewXZBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
|
||||
return NewCompressedBlobReader(
|
||||
reader,
|
||||
func(r io.Reader) (io.ReadCloser, error) {
|
||||
decoder, err := xz.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &XZBlobReader{decoder}, nil
|
||||
},
|
||||
clusterStartOffset,
|
||||
clusterEndOffset,
|
||||
blobIndex,
|
||||
blobSize,
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type ZstdBlobReader struct {
|
||||
decoder *zstd.Decoder
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *ZstdBlobReader) Close() error {
|
||||
r.decoder.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *ZstdBlobReader) Read(p []byte) (n int, err error) {
|
||||
return r.decoder.Read(p)
|
||||
}
|
||||
|
||||
var _ io.ReadCloser = &ZstdBlobReader{}
|
||||
|
||||
func NewZStdBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
|
||||
return NewCompressedBlobReader(
|
||||
reader,
|
||||
func(r io.Reader) (io.ReadCloser, error) {
|
||||
decoder, err := zstd.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &ZstdBlobReader{decoder}, nil
|
||||
},
|
||||
clusterStartOffset,
|
||||
clusterEndOffset,
|
||||
blobIndex,
|
||||
blobSize,
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,483 @@
|
|||
package bundle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bundle/zim"
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type ZimBundle struct {
|
||||
archivePath string
|
||||
|
||||
initOnce sync.Once
|
||||
initErr error
|
||||
|
||||
reader *zim.Reader
|
||||
urlNamespaceCache *lru.Cache[string, zim.Namespace]
|
||||
}
|
||||
|
||||
func (b *ZimBundle) File(filename string) (io.ReadCloser, os.FileInfo, error) {
|
||||
ctx := logger.With(
|
||||
context.Background(),
|
||||
logger.F("filename", filename),
|
||||
)
|
||||
|
||||
logger.Debug(ctx, "opening file")
|
||||
|
||||
switch filename {
|
||||
case "manifest.yml":
|
||||
return b.renderFakeManifest(ctx)
|
||||
case "server/main.js":
|
||||
return b.renderFakeServerMain(ctx)
|
||||
case "public":
|
||||
return b.renderDirectory(ctx, filename)
|
||||
case "public/index.html":
|
||||
return b.renderMainPage(ctx, filename)
|
||||
|
||||
default:
|
||||
return b.renderURL(ctx, filename)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *ZimBundle) Dir(dirname string) ([]os.FileInfo, error) {
|
||||
files := make([]os.FileInfo, 0)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderFakeManifest(ctx context.Context) (io.ReadCloser, os.FileInfo, error) {
|
||||
if err := b.init(); err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
metadata, err := b.reader.Metadata()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
manifest := map[string]any{}
|
||||
|
||||
manifest["version"] = "0.0.0"
|
||||
|
||||
if name, exists := metadata[zim.MetadataName]; exists {
|
||||
replacer := strings.NewReplacer(
|
||||
"_", "",
|
||||
" ", "",
|
||||
)
|
||||
|
||||
manifest["id"] = strings.ToLower(replacer.Replace(name)) + ".zim.edge.app"
|
||||
} else {
|
||||
manifest["id"] = b.reader.UUID() + ".zim.edge.app"
|
||||
}
|
||||
|
||||
if title, exists := metadata[zim.MetadataTitle]; exists {
|
||||
manifest["title"] = title
|
||||
} else {
|
||||
manifest["title"] = "Unknown"
|
||||
}
|
||||
|
||||
if description, exists := metadata[zim.MetadataDescription]; exists {
|
||||
manifest["description"] = description
|
||||
}
|
||||
|
||||
favicon, err := b.reader.Favicon()
|
||||
if err != nil && !errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if favicon != nil {
|
||||
manifestMeta, exists := manifest["metadata"].(map[string]any)
|
||||
if !exists {
|
||||
manifestMeta = make(map[string]any)
|
||||
manifest["metadata"] = manifestMeta
|
||||
}
|
||||
|
||||
paths, exists := manifestMeta["paths"].(map[string]any)
|
||||
if !exists {
|
||||
paths = make(map[string]any)
|
||||
manifestMeta["paths"] = paths
|
||||
}
|
||||
|
||||
paths["icon"] = "/" + favicon.FullURL()
|
||||
}
|
||||
|
||||
data, err := yaml.Marshal(manifest)
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
stat := &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: "manifest.yml",
|
||||
size: int64(len(data)),
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(data)
|
||||
file := io.NopCloser(buf)
|
||||
|
||||
return file, stat, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderFakeServerMain(ctx context.Context) (io.ReadCloser, os.FileInfo, error) {
|
||||
stat := &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: "server/main.js",
|
||||
size: 0,
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
file := io.NopCloser(buf)
|
||||
|
||||
return file, stat, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderURL(ctx context.Context, url string) (io.ReadCloser, os.FileInfo, error) {
|
||||
if err := b.init(); err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
url = strings.TrimPrefix(url, "public/")
|
||||
|
||||
entry, err := b.searchEntryFromURL(ctx, url)
|
||||
if err != nil {
|
||||
if errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
logger.Debug(
|
||||
ctx, "found zim entry",
|
||||
logger.F("webURL", url),
|
||||
logger.F("zimFullURL", entry.FullURL()),
|
||||
)
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
contentReader, err := content.Reader()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
size, err := contentReader.Size()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
filename := filepath.Base(url)
|
||||
|
||||
mimeType := content.MimeType()
|
||||
if mimeType != "text/html" {
|
||||
zimFile := &zimFile{
|
||||
fileInfo: &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: filename,
|
||||
size: size,
|
||||
},
|
||||
reader: contentReader,
|
||||
}
|
||||
|
||||
return zimFile, zimFile.fileInfo, nil
|
||||
}
|
||||
|
||||
// Read HTML file and inject Edge scripts
|
||||
|
||||
data, err := io.ReadAll(contentReader)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
injected, err := b.injectEdgeScriptTag(data)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not inject edge script", logger.E(errors.WithStack(err)))
|
||||
} else {
|
||||
data = injected
|
||||
}
|
||||
|
||||
zimFile := &zimFile{
|
||||
fileInfo: &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: filename,
|
||||
size: size,
|
||||
},
|
||||
reader: io.NopCloser(bytes.NewBuffer(data)),
|
||||
}
|
||||
|
||||
return zimFile, zimFile.fileInfo, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) searchEntryFromURL(ctx context.Context, url string) (zim.Entry, error) {
|
||||
ctx = logger.With(ctx, logger.F("webURL", url))
|
||||
|
||||
logger.Debug(ctx, "searching entry namespace in local cache")
|
||||
|
||||
entry, err := b.reader.EntryWithFullURL(url)
|
||||
if err != nil && !errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if entry != nil {
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
contentNamespaces := []zim.Namespace{
|
||||
zim.V6NamespaceContent,
|
||||
zim.V6NamespaceMetadata,
|
||||
zim.V5NamespaceLayout,
|
||||
zim.V5NamespaceArticle,
|
||||
zim.V5NamespaceImageFile,
|
||||
zim.V5NamespaceMetadata,
|
||||
}
|
||||
|
||||
logger.Debug(
|
||||
ctx, "make educated guesses about potential url namespace",
|
||||
logger.F("zimNamespaces", contentNamespaces),
|
||||
)
|
||||
|
||||
for _, ns := range contentNamespaces {
|
||||
logger.Debug(
|
||||
ctx, "trying to access entry directly",
|
||||
logger.F("zimNamespace", ns),
|
||||
logger.F("zimURL", url),
|
||||
)
|
||||
|
||||
entry, err := b.reader.EntryWithURL(ns, url)
|
||||
if err != nil && !errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if entry != nil {
|
||||
b.urlNamespaceCache.Add(url, entry.Namespace())
|
||||
return entry, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "doing full entries scan")
|
||||
|
||||
iterator := b.reader.Entries()
|
||||
for iterator.Next() {
|
||||
current := iterator.Entry()
|
||||
|
||||
if current.FullURL() != url && current.URL() != url {
|
||||
continue
|
||||
}
|
||||
|
||||
entry = current
|
||||
b.urlNamespaceCache.Add(url, entry.Namespace())
|
||||
break
|
||||
}
|
||||
if err := iterator.Err(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
return nil, errors.WithStack(zim.ErrNotFound)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderDirectory(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) {
|
||||
zimFile := &zimFile{
|
||||
fileInfo: &zimFileInfo{
|
||||
isDir: true,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: filename,
|
||||
size: 0,
|
||||
},
|
||||
reader: io.NopCloser(bytes.NewBuffer(nil)),
|
||||
}
|
||||
|
||||
return zimFile, zimFile.fileInfo, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderMainPage(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) {
|
||||
if err := b.init(); err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
main, err := b.reader.MainPage()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return b.renderURL(ctx, main.FullURL())
|
||||
}
|
||||
|
||||
func (b *ZimBundle) injectEdgeScriptTag(data []byte) ([]byte, error) {
|
||||
buff := bytes.NewBuffer(data)
|
||||
doc, err := html.Parse(buff)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var f func(*html.Node) bool
|
||||
f = func(n *html.Node) bool {
|
||||
if n.Type == html.ElementNode && n.Data == "head" {
|
||||
script := &html.Node{
|
||||
Type: html.ElementNode,
|
||||
Data: "script",
|
||||
Attr: []html.Attribute{
|
||||
{
|
||||
Key: "src",
|
||||
Val: "/edge/sdk/client.js",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
n.AppendChild(script)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if keepWalking := f(c); !keepWalking {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
f(doc)
|
||||
|
||||
buff.Reset()
|
||||
|
||||
if err := html.Render(buff, doc); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return buff.Bytes(), nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) init() error {
|
||||
b.initOnce.Do(func() {
|
||||
reader, err := zim.Open(b.archivePath)
|
||||
if err != nil {
|
||||
b.initErr = errors.Wrapf(err, "could not open '%v'", b.archivePath)
|
||||
return
|
||||
}
|
||||
|
||||
b.reader = reader
|
||||
|
||||
cache, err := lru.New[string, zim.Namespace](128)
|
||||
if err != nil {
|
||||
b.initErr = errors.Wrap(err, "could not initialize cache")
|
||||
return
|
||||
}
|
||||
|
||||
b.urlNamespaceCache = cache
|
||||
})
|
||||
if b.initErr != nil {
|
||||
return errors.WithStack(b.initErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewZimBundle(archivePath string) *ZimBundle {
|
||||
return &ZimBundle{
|
||||
archivePath: archivePath,
|
||||
}
|
||||
}
|
||||
|
||||
type zimFile struct {
|
||||
fileInfo *zimFileInfo
|
||||
reader io.ReadCloser
|
||||
}
|
||||
|
||||
// Close implements fs.File.
|
||||
func (f *zimFile) Close() error {
|
||||
if err := f.reader.Close(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements fs.File.
|
||||
func (f *zimFile) Read(d []byte) (int, error) {
|
||||
n, err := f.reader.Read(d)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
return n, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Stat implements fs.File.
|
||||
func (f *zimFile) Stat() (fs.FileInfo, error) {
|
||||
return f.fileInfo, nil
|
||||
}
|
||||
|
||||
var _ fs.File = &zimFile{}
|
||||
|
||||
type zimFileInfo struct {
|
||||
isDir bool
|
||||
modTime time.Time
|
||||
mode fs.FileMode
|
||||
name string
|
||||
size int64
|
||||
}
|
||||
|
||||
// IsDir implements fs.FileInfo.
|
||||
func (i *zimFileInfo) IsDir() bool {
|
||||
return i.isDir
|
||||
}
|
||||
|
||||
// ModTime implements fs.FileInfo.
|
||||
func (i *zimFileInfo) ModTime() time.Time {
|
||||
return i.modTime
|
||||
}
|
||||
|
||||
// Mode implements fs.FileInfo.
|
||||
func (i *zimFileInfo) Mode() fs.FileMode {
|
||||
return i.mode
|
||||
}
|
||||
|
||||
// Name implements fs.FileInfo.
|
||||
func (i *zimFileInfo) Name() string {
|
||||
return i.name
|
||||
}
|
||||
|
||||
// Size implements fs.FileInfo.
|
||||
func (i *zimFileInfo) Size() int64 {
|
||||
return i.size
|
||||
}
|
||||
|
||||
// Sys implements fs.FileInfo.
|
||||
func (*zimFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ fs.FileInfo = &zimFileInfo{}
|
||||
|
|
@ -27,7 +27,6 @@ func HTML5Fileserver(fs http.FileSystem) http.Handler {
|
|||
r.URL.Path = "/"
|
||||
|
||||
handler.ServeHTTP(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue