diff --git a/go.mod b/go.mod index 1468140..c01ed51 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,15 @@ module forge.cadoles.com/arcad/edge -go 1.19 +go 1.21 require ( github.com/getsentry/sentry-go v0.25.0 - github.com/hashicorp/golang-lru/v2 v2.0.6 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/mdns v1.0.5 github.com/keegancsmith/rpc v1.3.0 + github.com/klauspost/compress v1.16.6 github.com/lestrrat-go/jwx/v2 v2.0.8 + github.com/ulikunitz/xz v0.5.11 modernc.org/sqlite v1.20.4 ) @@ -62,12 +64,12 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648 go.opencensus.io v0.22.5 // indirect - golang.org/x/crypto v0.7.0 + golang.org/x/crypto v0.10.0 golang.org/x/mod v0.10.0 - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.11.0 + golang.org/x/sys v0.9.0 // indirect + golang.org/x/term v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/tools v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 5c26365..3ee3437 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxm github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -178,6 +179,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -194,8 +196,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/hashicorp/go.net v0.0.0-20151006203346-104dcad90073/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= -github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/mdns v0.0.0-20151206042412-9d85cf22f9f8/go.mod h1:aa76Av3qgPeIQp9Y3qIkTBPieQYNkQ13Kxe7pze9Wb0= github.com/hashicorp/mdns v1.0.5 h1:1M5hW1cunYeoXOqHwEb/GBDDHAFo0Yqb/uz/beC6LbE= @@ -212,6 +214,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/keegancsmith/rpc v1.3.0 h1:wGWOpjcNrZaY8GDYZJfvyxmlLljm3YQWF+p918DXtDk= github.com/keegancsmith/rpc v1.3.0/go.mod h1:6O2xnOGjPyvIPbvp0MdrOe5r6cu1GZ4JoTzpzDhWeo0= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= +github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -245,6 +249,7 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw= @@ -260,6 +265,7 @@ github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CF github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -293,6 +299,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0= github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= @@ -322,8 +330,9 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -398,8 +407,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -464,15 +473,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -484,8 +493,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -662,7 +671,9 @@ modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= @@ -676,9 +687,11 @@ modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34= +modernc.org/tcl v1.15.0/go.mod h1:xRoGotBZ6dU+Zo2tca+2EqVEeMmOUBzHnhIwq4YrVnE= modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= +modernc.org/z v1.7.0/go.mod h1:hVdgNMh8ggTuRG1rGU8x+xGRFfiQUIAw0ZqlPy8+HyQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/modd.conf b/modd.conf index 877a0c9..25a1b13 100644 --- a/modd.conf +++ b/modd.conf @@ -17,5 +17,5 @@ misc/client-sdk-testsuite/src/**/* } **/*.go { - prep: make GOTEST_ARGS="-short" test + # prep: make GOTEST_ARGS="-short" test } \ No newline at end of file diff --git a/pkg/app/option.go b/pkg/app/option.go new file mode 100644 index 0000000..8a8e8b1 --- /dev/null +++ b/pkg/app/option.go @@ -0,0 +1,36 @@ +package app + +import ( + "context" + + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" +) + +type Options struct { + ModuleFactories []ServerModuleFactory + ErrorHandler func(ctx context.Context, err error) +} + +type OptionFunc func(opts *Options) + +func NewOptions(funcs ...OptionFunc) *Options { + opts := &Options{ + ModuleFactories: make([]ServerModuleFactory, 0), + ErrorHandler: func(ctx context.Context, err error) { + logger.Error(ctx, err.Error(), logger.E(errors.WithStack(err))) + }, + } + + for _, fn := range funcs { + fn(opts) + } + + return opts +} + +func WithModulesFactories(factories ...ServerModuleFactory) OptionFunc { + return func(opts *Options) { + opts.ModuleFactories = factories + } +} diff --git a/pkg/bundle/from_path.go b/pkg/bundle/from_path.go index 89448fd..2b2f6ed 100644 --- a/pkg/bundle/from_path.go +++ b/pkg/bundle/from_path.go @@ -13,6 +13,7 @@ type ArchiveExt string const ( ExtZip ArchiveExt = "zip" ExtTarGz ArchiveExt = "tar.gz" + ExtZim ArchiveExt = "zim" ) func FromPath(path string) (Bundle, error) { @@ -56,5 +57,14 @@ func matchArchivePattern(archivePath string) (Bundle, error) { return NewZipBundle(archivePath), nil } + matches, err = filepath.Match(fmt.Sprintf("*.%s", ExtZim), base) + if err != nil { + return nil, errors.Wrapf(err, "could not match file archive '%s'", archivePath) + } + + if matches { + return NewZimBundle(archivePath), nil + } + return nil, errors.WithStack(ErrUnknownBundleArchiveExt) } diff --git a/pkg/bundle/zim/blob_reader.go b/pkg/bundle/zim/blob_reader.go new file mode 100644 index 0000000..452b3c1 --- /dev/null +++ b/pkg/bundle/zim/blob_reader.go @@ -0,0 +1,8 @@ +package zim + +import "io" + +type BlobReader interface { + io.ReadCloser + Size() (int64, error) +} diff --git a/pkg/bundle/zim/compressed_blob_reader.go b/pkg/bundle/zim/compressed_blob_reader.go new file mode 100644 index 0000000..0695b12 --- /dev/null +++ b/pkg/bundle/zim/compressed_blob_reader.go @@ -0,0 +1,163 @@ +package zim + +import ( + "bytes" + "encoding/binary" + "io" + "os" + "sync" + + "github.com/pkg/errors" +) + +type CompressedBlobReader struct { + reader *Reader + decoderFactory BlobDecoderFactory + + clusterStartOffset uint64 + clusterEndOffset uint64 + blobIndex uint32 + blobSize int + readOffset uint64 + + loadCluster sync.Once + loadClusterErr error + + data []byte + closed bool +} + +// Size implements BlobReader. +func (r *CompressedBlobReader) Size() (int64, error) { + if err := r.loadClusterData(); err != nil { + return 0, errors.WithStack(err) + } + + return int64(len(r.data)), nil +} + +// Close implements io.ReadCloser. +func (r *CompressedBlobReader) Close() error { + clear(r.data) + r.closed = true + return nil +} + +// Read implements io.ReadCloser. +func (r *CompressedBlobReader) Read(p []byte) (int, error) { + if err := r.loadClusterData(); err != nil { + return 0, errors.WithStack(err) + } + + length := len(p) + remaining := len(r.data) - int(r.readOffset) + if length > remaining { + length = remaining + } + + chunk := make([]byte, length) + + copy(chunk, r.data[r.readOffset:int(r.readOffset)+length]) + copy(p, chunk) + + if length == remaining { + return length, io.EOF + } + + r.readOffset += uint64(length) + + return length, nil +} + +func (r *CompressedBlobReader) loadClusterData() error { + if r.closed { + return errors.WithStack(os.ErrClosed) + } + + r.loadCluster.Do(func() { + compressedData := make([]byte, r.clusterEndOffset-r.clusterStartOffset) + if err := r.reader.readRange(int64(r.clusterStartOffset+1), compressedData); err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + blobBuffer := bytes.NewBuffer(compressedData) + + decoder, err := r.decoderFactory(blobBuffer) + if err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + defer decoder.Close() + + uncompressedData, err := io.ReadAll(decoder) + if err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + var ( + blobStart uint64 + blobEnd uint64 + ) + + if r.blobSize == 8 { + blobStart64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian) + if err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + blobStart = blobStart64 + + blobEnd64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian) + if err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + blobEnd = blobEnd64 + } else { + blobStart32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian) + if err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + blobStart = uint64(blobStart32) + + blobEnd32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian) + if err != nil { + r.loadClusterErr = errors.WithStack(err) + return + } + + blobEnd = uint64(blobEnd32) + } + + r.data = make([]byte, blobEnd-blobStart) + copy(r.data, uncompressedData[blobStart:blobEnd]) + }) + if r.loadClusterErr != nil { + return errors.WithStack(r.loadClusterErr) + } + + return nil +} + +type BlobDecoderFactory func(io.Reader) (io.ReadCloser, error) + +func NewCompressedBlobReader(reader *Reader, decoderFactory BlobDecoderFactory, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader { + return &CompressedBlobReader{ + reader: reader, + decoderFactory: decoderFactory, + clusterStartOffset: clusterStartOffset, + clusterEndOffset: clusterEndOffset, + blobIndex: blobIndex, + blobSize: blobSize, + readOffset: 0, + } +} + +var _ BlobReader = &UncompressedBlobReader{} diff --git a/pkg/bundle/zim/content_entry.go b/pkg/bundle/zim/content_entry.go new file mode 100644 index 0000000..3b48aba --- /dev/null +++ b/pkg/bundle/zim/content_entry.go @@ -0,0 +1,193 @@ +package zim + +import ( + "encoding/binary" + + "github.com/pkg/errors" +) + +type zimCompression int + +const ( + zimCompressionNoneZeno zimCompression = 0 + zimCompressionNone zimCompression = 1 + zimCompressionNoneZLib zimCompression = 2 + zimCompressionNoneBZip2 zimCompression = 3 + zimCompressionNoneXZ zimCompression = 4 + zimCompressionNoneZStandard zimCompression = 5 +) + +type ContentEntry struct { + *BaseEntry + mimeType string + clusterIndex uint32 + blobIndex uint32 +} + +func (e *ContentEntry) Compression() (int, error) { + clusterHeader, _, _, err := e.readClusterInfo() + if err != nil { + return 0, errors.WithStack(err) + } + + return int((clusterHeader << 4) >> 4), nil +} + +func (e *ContentEntry) MimeType() string { + return e.mimeType +} + +func (e *ContentEntry) Reader() (BlobReader, error) { + clusterHeader, clusterStartOffset, clusterEndOffset, err := e.readClusterInfo() + if err != nil { + return nil, errors.WithStack(err) + } + + compression := (clusterHeader << 4) >> 4 + extended := (clusterHeader<<3)>>7 == 1 + + blobSize := 4 + if extended { + blobSize = 8 + } + + switch compression { + + // Uncompressed blobs + case uint8(zimCompressionNoneZeno): + fallthrough + case uint8(zimCompressionNone): + startPos := clusterStartOffset + 1 + blobOffset := uint64(e.blobIndex * uint32(blobSize)) + + data := make([]byte, 2*blobSize) + if err := e.reader.readRange(int64(startPos+blobOffset), data); err != nil { + return nil, errors.WithStack(err) + } + + var ( + blobStart uint64 + blobEnd uint64 + ) + + if extended { + blobStart64, err := readUint64(data[0:blobSize], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + blobStart = blobStart64 + + blobEnd64, err := readUint64(data[blobSize:blobSize*2], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + blobEnd = uint64(blobEnd64) + } else { + blobStart32, err := readUint32(data[0:blobSize], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + blobStart = uint64(blobStart32) + + blobEnd32, err := readUint32(data[blobSize:blobSize*2], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + blobEnd = uint64(blobEnd32) + } + + return NewUncompressedBlobReader(e.reader, startPos+blobStart, startPos+blobEnd, blobSize), nil + + // Supported compression algorithms + case uint8(zimCompressionNoneXZ): + return NewXZBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil + + case uint8(zimCompressionNoneZStandard): + return NewZStdBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil + + // Unsupported compression algorithms + case uint8(zimCompressionNoneZLib): + fallthrough + case uint8(zimCompressionNoneBZip2): + fallthrough + default: + return nil, errors.Wrapf(ErrCompressionAlgorithmNotSupported, "unexpected compression algorithm '%d'", compression) + } +} + +func (e *ContentEntry) Redirect() (*ContentEntry, error) { + return e, nil +} + +func (e *ContentEntry) readClusterInfo() (uint8, uint64, uint64, error) { + startClusterOffset, clusterEndOffset, err := e.reader.getClusterOffsets(int(e.clusterIndex)) + if err != nil { + return 0, 0, 0, errors.WithStack(err) + } + + data := make([]byte, 1) + if err := e.reader.readRange(int64(startClusterOffset), data); err != nil { + return 0, 0, 0, errors.WithStack(err) + } + + clusterHeader := uint8(data[0]) + + return clusterHeader, startClusterOffset, clusterEndOffset, nil +} + +func (r *Reader) parseContentEntry(offset int64, base *BaseEntry) (*ContentEntry, error) { + entry := &ContentEntry{ + BaseEntry: base, + } + + data := make([]byte, 16) + if err := r.readRange(offset, data); err != nil { + return nil, errors.WithStack(err) + } + + mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + if mimeTypeIndex >= uint16(len(r.mimeTypes)) { + return nil, errors.Errorf("mime type index '%d' greater than mime types length '%d'", mimeTypeIndex, len(r.mimeTypes)) + } + + entry.mimeType = r.mimeTypes[mimeTypeIndex] + + entry.namespace = Namespace(data[3:4]) + + clusterIndex, err := readUint32(data[8:12], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + entry.clusterIndex = clusterIndex + + blobIndex, err := readUint32(data[12:16], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + entry.blobIndex = blobIndex + + strs, _, err := r.readStringsAt(offset+16, 2, 1024) + if err != nil { + return nil, errors.WithStack(err) + } + + if len(strs) > 0 { + entry.url = strs[0] + } + + if len(strs) > 1 { + entry.title = strs[1] + } + + return entry, nil +} diff --git a/pkg/bundle/zim/entry.go b/pkg/bundle/zim/entry.go new file mode 100644 index 0000000..bd58d16 --- /dev/null +++ b/pkg/bundle/zim/entry.go @@ -0,0 +1,135 @@ +package zim + +import ( + "encoding/binary" + "fmt" + + "github.com/pkg/errors" +) + +type Entry interface { + Redirect() (*ContentEntry, error) + Namespace() Namespace + URL() string + FullURL() string + Title() string +} + +type BaseEntry struct { + mimeTypeIndex uint16 + namespace Namespace + url string + title string + reader *Reader +} + +func (e *BaseEntry) Namespace() Namespace { + return e.namespace +} + +func (e *BaseEntry) Title() string { + if e.title == "" { + return e.url + } + + return e.title +} + +func (e *BaseEntry) URL() string { + return e.url +} + +func (e *BaseEntry) FullURL() string { + return toFullURL(e.Namespace(), e.URL()) +} + +func (r *Reader) parseBaseEntry(offset int64) (*BaseEntry, error) { + entry := &BaseEntry{ + reader: r, + } + + data := make([]byte, 3) + if err := r.readRange(offset, data); err != nil { + return nil, errors.WithStack(err) + } + + mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + entry.mimeTypeIndex = mimeTypeIndex + entry.namespace = Namespace(data[2]) + + return entry, nil +} + +type RedirectEntry struct { + *BaseEntry + redirectIndex uint32 +} + +func (e *RedirectEntry) Redirect() (*ContentEntry, error) { + if e.redirectIndex >= uint32(len(e.reader.urlIndex)) { + return nil, errors.Wrapf(ErrInvalidIndex, "entry index '%d' out of bounds", e.redirectIndex) + } + + entryPtr := e.reader.urlIndex[e.redirectIndex] + entry, err := e.reader.parseEntryAt(int64(entryPtr)) + if err != nil { + return nil, errors.WithStack(err) + } + + entry, err = entry.Redirect() + if err != nil { + return nil, errors.WithStack(err) + } + + contentEntry, ok := entry.(*ContentEntry) + if !ok { + return nil, errors.WithStack(ErrInvalidRedirect) + } + + return contentEntry, nil +} + +func (r *Reader) parseRedirectEntry(offset int64, base *BaseEntry) (*RedirectEntry, error) { + entry := &RedirectEntry{ + BaseEntry: base, + } + + data := make([]byte, 4) + if err := r.readRange(offset+8, data); err != nil { + return nil, errors.WithStack(err) + } + + redirectIndex, err := readUint32(data, binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + entry.redirectIndex = redirectIndex + + strs, _, err := r.readStringsAt(offset+12, 2, 1024) + if err != nil { + return nil, errors.WithStack(err) + } + + if len(strs) > 0 { + entry.url = strs[0] + } + + if len(strs) > 1 { + entry.title = strs[1] + } + + return entry, nil +} + +func toFullURL(ns Namespace, url string) string { + if ns == "\x00" { + return url + } + + return fmt.Sprintf("%s/%s", ns, url) +} diff --git a/pkg/bundle/zim/entry_iterator.go b/pkg/bundle/zim/entry_iterator.go new file mode 100644 index 0000000..13622fc --- /dev/null +++ b/pkg/bundle/zim/entry_iterator.go @@ -0,0 +1,46 @@ +package zim + +import "github.com/pkg/errors" + +type EntryIterator struct { + index int + entry Entry + err error + reader *Reader +} + +func (it *EntryIterator) Next() bool { + if it.err != nil { + return false + } + + entryCount := it.reader.EntryCount() + + if it.index >= int(entryCount-1) { + return false + } + + entry, err := it.reader.EntryAt(it.index) + if err != nil { + it.err = errors.WithStack(err) + + return false + } + + it.entry = entry + it.index++ + + return true +} + +func (it *EntryIterator) Err() error { + return it.err +} + +func (it *EntryIterator) Index() int { + return it.index - 1 +} + +func (it *EntryIterator) Entry() Entry { + return it.entry +} diff --git a/pkg/bundle/zim/error.go b/pkg/bundle/zim/error.go new file mode 100644 index 0000000..35f5bf8 --- /dev/null +++ b/pkg/bundle/zim/error.go @@ -0,0 +1,10 @@ +package zim + +import "errors" + +var ( + ErrInvalidIndex = errors.New("invalid index") + ErrNotFound = errors.New("not found") + ErrInvalidRedirect = errors.New("invalid redirect") + ErrCompressionAlgorithmNotSupported = errors.New("compression algorithm not supported") +) diff --git a/pkg/bundle/zim/favicon.go b/pkg/bundle/zim/favicon.go new file mode 100644 index 0000000..d4e8776 --- /dev/null +++ b/pkg/bundle/zim/favicon.go @@ -0,0 +1,66 @@ +package zim + +import "github.com/pkg/errors" + +func (r *Reader) Favicon() (*ContentEntry, error) { + illustration, err := r.getMetadataIllustration() + if err != nil && !errors.Is(err, ErrNotFound) { + return nil, errors.WithStack(err) + } + + if illustration != nil { + return illustration, nil + } + + namespaces := []Namespace{V5NamespaceLayout, V5NamespaceImageFile} + urls := []string{"favicon", "favicon.png"} + + for _, ns := range namespaces { + for _, url := range urls { + entry, err := r.EntryWithURL(ns, url) + if err != nil && !errors.Is(err, ErrNotFound) { + return nil, errors.WithStack(err) + } + + if errors.Is(err, ErrNotFound) { + continue + } + + content, err := entry.Redirect() + if err != nil { + return nil, errors.WithStack(err) + } + + return content, nil + } + } + + return nil, errors.WithStack(ErrNotFound) +} + +func (r *Reader) getMetadataIllustration() (*ContentEntry, error) { + keys := []MetadataKey{MetadataIllustration96x96at2, MetadataIllustration48x48at1} + + metadata, err := r.Metadata(keys...) + if err != nil { + return nil, errors.WithStack(err) + } + + for _, k := range keys { + if _, exists := metadata[k]; exists { + entry, err := r.EntryWithURL(V5NamespaceMetadata, string(k)) + if err != nil { + return nil, errors.WithStack(err) + } + + content, err := entry.Redirect() + if err != nil { + return nil, errors.WithStack(err) + } + + return content, nil + } + } + + return nil, errors.WithStack(ErrNotFound) +} diff --git a/pkg/bundle/zim/metadata.go b/pkg/bundle/zim/metadata.go new file mode 100644 index 0000000..8b27e50 --- /dev/null +++ b/pkg/bundle/zim/metadata.go @@ -0,0 +1,81 @@ +package zim + +import ( + "io" + + "github.com/pkg/errors" +) + +type MetadataKey string + +// See https://wiki.openzim.org/wiki/Metadata +const ( + MetadataName MetadataKey = "Name" + MetadataTitle MetadataKey = "Title" + MetadataDescription MetadataKey = "Description" + MetadataLongDescription MetadataKey = "LongDescription" + MetadataCreator MetadataKey = "Creator" + MetadataTags MetadataKey = "Tags" + MetadataDate MetadataKey = "Date" + MetadataPublisher MetadataKey = "Publisher" + MetadataFlavour MetadataKey = "Flavour" + MetadataSource MetadataKey = "Source" + MetadataLanguage MetadataKey = "Language" + MetadataIllustration48x48at1 MetadataKey = "Illustration_48x48@1" + MetadataIllustration96x96at2 MetadataKey = "Illustration_96x96@2" +) + +var knownKeys = []MetadataKey{ + MetadataName, + MetadataTitle, + MetadataDescription, + MetadataLongDescription, + MetadataCreator, + MetadataPublisher, + MetadataLanguage, + MetadataTags, + MetadataDate, + MetadataFlavour, + MetadataSource, + MetadataIllustration48x48at1, + MetadataIllustration96x96at2, +} + +// Metadata returns a copy of the internal metadata map of the ZIM file. +func (r *Reader) Metadata(keys ...MetadataKey) (map[MetadataKey]string, error) { + if len(keys) == 0 { + keys = knownKeys + } + + metadata := make(map[MetadataKey]string) + + for _, key := range keys { + entry, err := r.EntryWithURL(V5NamespaceMetadata, string(key)) + if err != nil { + if errors.Is(err, ErrNotFound) { + continue + } + + return nil, errors.WithStack(err) + } + + content, err := entry.Redirect() + if err != nil { + return nil, errors.WithStack(err) + } + + reader, err := content.Reader() + if err != nil { + return nil, errors.WithStack(err) + } + + data, err := io.ReadAll(reader) + if err != nil { + return nil, errors.WithStack(err) + } + + metadata[key] = string(data) + } + + return metadata, nil +} diff --git a/pkg/bundle/zim/namespace.go b/pkg/bundle/zim/namespace.go new file mode 100644 index 0000000..db869db --- /dev/null +++ b/pkg/bundle/zim/namespace.go @@ -0,0 +1,23 @@ +package zim + +type Namespace string + +const ( + V6NamespaceContent Namespace = "C" + V6NamespaceMetadata Namespace = "M" + V6NamespaceWellKnown Namespace = "W" + V6NamespaceSearch Namespace = "X" +) + +const ( + V5NamespaceLayout Namespace = "-" + V5NamespaceArticle Namespace = "A" + V5NamespaceArticleMetadata Namespace = "B" + V5NamespaceImageFile Namespace = "I" + V5NamespaceImageText Namespace = "J" + V5NamespaceMetadata Namespace = "M" + V5NamespaceCategoryText Namespace = "U" + V5NamespaceCategoryArticleList Namespace = "V" + V5NamespaceCategoryPerArticle Namespace = "W" + V5NamespaceSearch Namespace = "X" +) diff --git a/pkg/bundle/zim/option.go b/pkg/bundle/zim/option.go new file mode 100644 index 0000000..ee0677f --- /dev/null +++ b/pkg/bundle/zim/option.go @@ -0,0 +1,30 @@ +package zim + +import "time" + +type Options struct { + URLCacheSize int + URLCacheTTL time.Duration + CacheSize int +} + +type OptionFunc func(opts *Options) + +func NewOptions(funcs ...OptionFunc) *Options { + funcs = append([]OptionFunc{ + WithCacheSize(2048), + }, funcs...) + + opts := &Options{} + for _, fn := range funcs { + fn(opts) + } + + return opts +} + +func WithCacheSize(size int) OptionFunc { + return func(opts *Options) { + opts.CacheSize = size + } +} diff --git a/pkg/bundle/zim/reader.go b/pkg/bundle/zim/reader.go new file mode 100644 index 0000000..560121b --- /dev/null +++ b/pkg/bundle/zim/reader.go @@ -0,0 +1,558 @@ +package zim + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "os" + "strings" + + lru "github.com/hashicorp/golang-lru/v2" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" +) + +const zimFormatMagicNumber uint32 = 0x44D495A +const nullByte = '\x00' +const zimRedirect = 0xffff + +type Reader struct { + majorVersion uint16 + minorVersion uint16 + uuid string + entryCount uint32 + clusterCount uint32 + urlPtrPos uint64 + titlePtrPos uint64 + clusterPtrPos uint64 + mimeListPos uint64 + mainPage uint32 + layoutPage uint32 + checksumPos uint64 + + mimeTypes []string + urlIndex []uint64 + clusterIndex []uint64 + + cache *lru.Cache[string, Entry] + urls map[string]int + + rangeReader RangeReadCloser +} + +func (r *Reader) Version() (majorVersion, minorVersion uint16) { + return r.majorVersion, r.minorVersion +} + +func (r *Reader) EntryCount() uint32 { + return r.entryCount +} + +func (r *Reader) ClusterCount() uint32 { + return r.clusterCount +} + +func (r *Reader) UUID() string { + return r.uuid +} + +func (r *Reader) Close() error { + if err := r.rangeReader.Close(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func (r *Reader) MainPage() (Entry, error) { + if r.mainPage == 0xffffffff { + return nil, errors.WithStack(ErrNotFound) + } + + entry, err := r.EntryAt(int(r.mainPage)) + if err != nil { + return nil, errors.WithStack(ErrNotFound) + } + + return entry, nil +} + +func (r *Reader) Entries() *EntryIterator { + return &EntryIterator{ + reader: r, + } +} + +func (r *Reader) EntryAt(idx int) (Entry, error) { + if idx >= len(r.urlIndex) || idx < 0 { + return nil, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", idx) + } + + entryPtr := r.urlIndex[idx] + + entry, err := r.parseEntryAt(int64(entryPtr)) + if err != nil { + return nil, errors.WithStack(err) + } + + r.cacheEntry(entryPtr, entry) + + return entry, nil +} + +func (r *Reader) EntryWithFullURL(url string) (Entry, error) { + urlNum, exists := r.urls[url] + if !exists { + return nil, errors.WithStack(ErrNotFound) + } + + entry, err := r.EntryAt(urlNum) + if err != nil { + return nil, errors.WithStack(err) + } + + return entry, nil +} + +func (r *Reader) EntryWithURL(ns Namespace, url string) (Entry, error) { + fullURL := toFullURL(ns, url) + + entry, err := r.EntryWithFullURL(fullURL) + if err != nil { + return nil, errors.WithStack(err) + } + + return entry, nil +} + +func (r *Reader) EntryWithTitle(ns Namespace, title string) (Entry, error) { + entry, found := r.getEntryByTitleFromCache(ns, title) + if found { + logger.Debug(context.Background(), "found entry with title from cache", logger.F("entry", entry.FullURL())) + return entry, nil + } + + iterator := r.Entries() + + for iterator.Next() { + entry := iterator.Entry() + + if entry.Title() == title && entry.Namespace() == ns { + return entry, nil + } + } + if err := iterator.Err(); err != nil { + return nil, errors.WithStack(err) + } + + return nil, errors.WithStack(ErrNotFound) +} + +func (r *Reader) getURLCacheKey(fullURL string) string { + return "url:" + fullURL +} + +func (r *Reader) getTitleCacheKey(ns Namespace, title string) string { + return fmt.Sprintf("title:%s/%s", ns, title) +} + +func (r *Reader) cacheEntry(offset uint64, entry Entry) { + urlKey := r.getURLCacheKey(entry.FullURL()) + titleKey := r.getTitleCacheKey(entry.Namespace(), entry.Title()) + + _, urlFound := r.cache.Peek(urlKey) + _, titleFound := r.cache.Peek(titleKey) + + if urlFound && titleFound { + return + } + + r.cache.Add(urlKey, entry) + r.cache.Add(titleKey, entry) +} + +func (r *Reader) getEntryByTitleFromCache(namespace Namespace, title string) (Entry, bool) { + key := r.getTitleCacheKey(namespace, title) + return r.cache.Get(key) +} + +func (r *Reader) parse() error { + if err := r.parseHeader(); err != nil { + return errors.WithStack(err) + } + + if err := r.parseMimeTypes(); err != nil { + return errors.WithStack(err) + } + + if err := r.parseURLIndex(); err != nil { + return errors.WithStack(err) + } + + if err := r.parseClusterIndex(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func (r *Reader) parseHeader() error { + header := make([]byte, 80) + if err := r.readRange(0, header); err != nil { + return errors.WithStack(err) + } + + magicNumber, err := readUint32(header[0:4], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + if magicNumber != zimFormatMagicNumber { + return errors.Errorf("invalid zim magic number '%d'", magicNumber) + } + + majorVersion, err := readUint16(header[4:6], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.majorVersion = majorVersion + + minorVersion, err := readUint16(header[6:8], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.minorVersion = minorVersion + + if err := r.parseUUID(header[8:16]); err != nil { + return errors.WithStack(err) + } + + entryCount, err := readUint32(header[24:28], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.entryCount = entryCount + + clusterCount, err := readUint32(header[28:32], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.clusterCount = clusterCount + + urlPtrPos, err := readUint64(header[32:40], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.urlPtrPos = urlPtrPos + + titlePtrPos, err := readUint64(header[40:48], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.titlePtrPos = titlePtrPos + + clusterPtrPos, err := readUint64(header[48:56], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.clusterPtrPos = clusterPtrPos + + mimeListPos, err := readUint64(header[56:64], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.mimeListPos = mimeListPos + + mainPage, err := readUint32(header[64:68], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.mainPage = mainPage + + layoutPage, err := readUint32(header[68:72], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.layoutPage = layoutPage + + checksumPos, err := readUint64(header[72:80], binary.LittleEndian) + if err != nil { + return errors.WithStack(err) + } + + r.checksumPos = checksumPos + + return nil +} + +func (r *Reader) parseUUID(data []byte) error { + parts := make([]string, 0, 5) + + val32, err := readUint32(data[0:4], binary.BigEndian) + if err != nil { + return errors.WithStack(err) + } + + parts = append(parts, fmt.Sprintf("%08x", val32)) + + val16, err := readUint16(data[4:6], binary.BigEndian) + if err != nil { + return errors.WithStack(err) + } + + parts = append(parts, fmt.Sprintf("%04x", val16)) + + val16, err = readUint16(data[6:8], binary.BigEndian) + if err != nil { + return errors.WithStack(err) + } + + parts = append(parts, fmt.Sprintf("%04x", val16)) + + val16, err = readUint16(data[8:10], binary.BigEndian) + if err != nil { + return errors.WithStack(err) + } + + parts = append(parts, fmt.Sprintf("%04x", val16)) + + val32, err = readUint32(data[10:14], binary.BigEndian) + if err != nil { + return errors.WithStack(err) + } + + val16, err = readUint16(data[14:16], binary.BigEndian) + if err != nil { + return errors.WithStack(err) + } + + parts = append(parts, fmt.Sprintf("%x%x", val32, val16)) + + r.uuid = strings.Join(parts, "-") + + return nil +} + +func (r *Reader) parseMimeTypes() error { + mimeTypes := make([]string, 0) + offset := int64(r.mimeListPos) + read := int64(0) + var err error + var found []string + for { + found, read, err = r.readStringsAt(offset+read, 64, 1024) + if err != nil && !errors.Is(err, io.EOF) { + return errors.WithStack(err) + } + + if len(found) == 0 || found[0] == "" { + break + } + + mimeTypes = append(mimeTypes, found...) + } + + r.mimeTypes = mimeTypes + + return nil +} + +func (r *Reader) parseURLIndex() error { + urlIndex, err := r.parsePointerIndex(int64(r.urlPtrPos), int64(r.entryCount)) + if err != nil { + return errors.WithStack(err) + } + + r.urlIndex = urlIndex + + return nil +} + +func (r *Reader) parseClusterIndex() error { + clusterIndex, err := r.parsePointerIndex(int64(r.clusterPtrPos), int64(r.clusterCount+1)) + if err != nil { + return errors.WithStack(err) + } + + r.clusterIndex = clusterIndex + + return nil +} + +func (r *Reader) parseEntryAt(offset int64) (Entry, error) { + base, err := r.parseBaseEntry(offset) + if err != nil { + return nil, errors.WithStack(err) + } + + var entry Entry + + if base.mimeTypeIndex == zimRedirect { + entry, err = r.parseRedirectEntry(offset, base) + if err != nil { + return nil, errors.WithStack(err) + } + } else { + entry, err = r.parseContentEntry(offset, base) + if err != nil { + return nil, errors.WithStack(err) + } + } + + return entry, nil +} + +func (r *Reader) parsePointerIndex(startAddr int64, count int64) ([]uint64, error) { + index := make([]uint64, count) + + data := make([]byte, count*8) + if err := r.readRange(startAddr, data); err != nil { + return nil, errors.WithStack(err) + } + + for i := int64(0); i < count; i++ { + offset := i * 8 + ptr, err := readUint64(data[offset:offset+8], binary.LittleEndian) + if err != nil { + return nil, errors.WithStack(err) + } + + index[i] = ptr + } + + return index, nil +} + +func (r *Reader) getClusterOffsets(clusterNum int) (uint64, uint64, error) { + if clusterNum > len(r.clusterIndex)-1 || clusterNum < 0 { + return 0, 0, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", clusterNum) + } + + return r.clusterIndex[clusterNum], r.clusterIndex[clusterNum+1] - 1, nil +} + +func (r *Reader) preload() error { + r.urls = make(map[string]int, r.entryCount) + + iterator := r.Entries() + for iterator.Next() { + entry := iterator.Entry() + r.urls[entry.FullURL()] = iterator.Index() + } + if err := iterator.Err(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func (r *Reader) readRange(offset int64, v []byte) error { + read, err := r.rangeReader.ReadAt(v, offset) + if err != nil { + return errors.WithStack(err) + } + + if read != len(v) { + return io.EOF + } + + return nil +} + +func (r *Reader) readStringsAt(offset int64, count int, bufferSize int) ([]string, int64, error) { + var sb strings.Builder + read := int64(0) + + values := make([]string, 0, count) + wasNullByte := false + + for { + data := make([]byte, bufferSize) + err := r.readRange(offset+read, data) + if err != nil && !errors.Is(err, io.EOF) { + return nil, read, errors.WithStack(err) + } + + for idx := 0; idx < len(data); idx++ { + d := data[idx] + if err := sb.WriteByte(d); err != nil { + return nil, read, errors.WithStack(err) + } + + read++ + + if d == nullByte { + if wasNullByte { + return values, read, nil + } + + wasNullByte = true + + str := strings.TrimRight(sb.String(), "\x00") + values = append(values, str) + + if len(values) == count || errors.Is(err, io.EOF) { + return values, read, nil + } + + sb.Reset() + } else { + wasNullByte = false + } + } + } +} + +type RangeReadCloser interface { + io.Closer + ReadAt(data []byte, offset int64) (n int, err error) +} + +func NewReader(rangeReader RangeReadCloser, funcs ...OptionFunc) (*Reader, error) { + opts := NewOptions(funcs...) + + cache, err := lru.New[string, Entry](opts.CacheSize) + if err != nil { + return nil, errors.WithStack(err) + } + + reader := &Reader{ + rangeReader: rangeReader, + cache: cache, + } + + if err := reader.parse(); err != nil { + return nil, errors.WithStack(err) + } + + if err := reader.preload(); err != nil { + return nil, errors.WithStack(err) + } + + return reader, nil +} + +func Open(path string, funcs ...OptionFunc) (*Reader, error) { + file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm) + if err != nil { + return nil, errors.WithStack(err) + } + + reader, err := NewReader(file, funcs...) + if err != nil { + return nil, errors.WithStack(err) + } + + return reader, nil +} diff --git a/pkg/bundle/zim/reader_test.go b/pkg/bundle/zim/reader_test.go new file mode 100644 index 0000000..b55321b --- /dev/null +++ b/pkg/bundle/zim/reader_test.go @@ -0,0 +1,133 @@ +package zim + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" +) + +type readerTestCase struct { + UUID string `json:"uuid"` + EntryCount uint32 `json:"entryCount"` + Entries []struct { + Namespace Namespace `json:"namespace"` + URL string `json:"url"` + Size int64 `json:"size"` + Compression int `json:"compression"` + MimeType string `json:"mimeType"` + Title string `json:"title"` + } `json:"entries"` +} + +func TestReader(t *testing.T) { + if testing.Verbose() { + logger.SetLevel(logger.LevelDebug) + logger.SetFormat(logger.FormatHuman) + } + + files, err := filepath.Glob("testdata/*.zim") + if err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + for _, zf := range files { + testName := filepath.Base(zf) + testCase, err := loadZimFileTestCase(zf) + if err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + t.Run(testName, func(t *testing.T) { + reader, err := Open(zf) + if err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + defer func() { + if err := reader.Close(); err != nil { + t.Errorf("%+v", errors.WithStack(err)) + } + }() + + if e, g := testCase.UUID, reader.UUID(); e != g { + t.Errorf("reader.UUID(): expected '%s', got '%s'", e, g) + } + + if e, g := testCase.EntryCount, reader.EntryCount(); e != g { + t.Errorf("reader.EntryCount(): expected '%v', got '%v'", e, g) + } + + if testCase.Entries == nil { + return + } + + for _, entryTestCase := range testCase.Entries { + testName := fmt.Sprintf("Entry/%s/%s", entryTestCase.Namespace, entryTestCase.URL) + t.Run(testName, func(t *testing.T) { + entry, err := reader.EntryWithURL(entryTestCase.Namespace, entryTestCase.URL) + if err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + content, err := entry.Redirect() + if err != nil { + t.Errorf("%+v", errors.WithStack(err)) + } + + if e, g := entryTestCase.MimeType, content.MimeType(); e != g { + t.Errorf("content.MimeType(): expected '%v', got '%v'", e, g) + } + + if e, g := entryTestCase.Title, content.Title(); e != g { + t.Errorf("content.Title(): expected '%v', got '%v'", e, g) + } + + compression, err := content.Compression() + if err != nil { + t.Errorf("%+v", errors.WithStack(err)) + } + + if e, g := entryTestCase.Compression, compression; e != g { + t.Errorf("content.Compression(): expected '%v', got '%v'", e, g) + } + + contentReader, err := content.Reader() + if err != nil { + t.Errorf("%+v", errors.WithStack(err)) + } + + size, err := contentReader.Size() + if err != nil { + t.Errorf("%+v", errors.WithStack(err)) + } + + if e, g := entryTestCase.Size, size; e != g { + t.Errorf("content.Size(): expected '%v', got '%v'", e, g) + } + }) + } + }) + } +} + +func loadZimFileTestCase(zimFile string) (*readerTestCase, error) { + testCaseFile, _ := strings.CutSuffix(zimFile, ".zim") + + data, err := os.ReadFile(testCaseFile + ".json") + if err != nil { + return nil, errors.WithStack(err) + } + + testCase := &readerTestCase{} + if err := json.Unmarshal(data, testCase); err != nil { + return nil, errors.WithStack(err) + } + + return testCase, nil +} diff --git a/pkg/bundle/zim/testdata/beer.stackexchange.com_en_all_2023-05.json b/pkg/bundle/zim/testdata/beer.stackexchange.com_en_all_2023-05.json new file mode 100644 index 0000000..f83cb59 --- /dev/null +++ b/pkg/bundle/zim/testdata/beer.stackexchange.com_en_all_2023-05.json @@ -0,0 +1,14 @@ +{ + "uuid": "8d141c3b-115d-bf73-294a-ee3c2e6b97b0", + "entryCount": 6223, + "entries": [ + { + "namespace": "C", + "url": "users_page=9", + "compression": 5, + "size": 58646, + "mimeType": "text/html", + "title": "users_page=9" + } + ] +} \ No newline at end of file diff --git a/pkg/bundle/zim/testdata/beer.stackexchange.com_en_all_2023-05.zim b/pkg/bundle/zim/testdata/beer.stackexchange.com_en_all_2023-05.zim new file mode 100644 index 0000000..0fd2e28 Binary files /dev/null and b/pkg/bundle/zim/testdata/beer.stackexchange.com_en_all_2023-05.zim differ diff --git a/pkg/bundle/zim/testdata/cadoles.json b/pkg/bundle/zim/testdata/cadoles.json new file mode 100644 index 0000000..c9ea8a1 --- /dev/null +++ b/pkg/bundle/zim/testdata/cadoles.json @@ -0,0 +1,22 @@ +{ + "uuid": "cf81f094-d802-c790-b854-c74ad9701ddb", + "entryCount": 271, + "entries": [ + { + "namespace": "C", + "url": "blog/202206-ShowroomInnovation.jpg", + "compression": 1, + "size": 260260, + "mimeType": "image/jpeg", + "title": "blog/202206-ShowroomInnovation.jpg" + }, + { + "namespace": "C", + "url": "team/index.html", + "compression": 5, + "size": 93185, + "mimeType": "text/html", + "title": "Cadoles - Notre équipe" + } + ] +} \ No newline at end of file diff --git a/pkg/bundle/zim/testdata/cadoles.zim b/pkg/bundle/zim/testdata/cadoles.zim new file mode 100644 index 0000000..fac4912 Binary files /dev/null and b/pkg/bundle/zim/testdata/cadoles.zim differ diff --git a/pkg/bundle/zim/testdata/wikibooks_af_all_maxi_2023-06.json b/pkg/bundle/zim/testdata/wikibooks_af_all_maxi_2023-06.json new file mode 100644 index 0000000..1d9abf5 --- /dev/null +++ b/pkg/bundle/zim/testdata/wikibooks_af_all_maxi_2023-06.json @@ -0,0 +1,14 @@ +{ + "uuid": "ad4f406c-2021-2db8-c729-297568bbe376", + "entryCount": 330, + "entries": [ + { + "namespace": "M", + "url": "Illustration_48x48@1", + "compression": 5, + "size": 5365, + "mimeType": "text/plain", + "title": "Illustration_48x48@1" + } + ] +} \ No newline at end of file diff --git a/pkg/bundle/zim/testdata/wikibooks_af_all_maxi_2023-06.zim b/pkg/bundle/zim/testdata/wikibooks_af_all_maxi_2023-06.zim new file mode 100644 index 0000000..558e37b Binary files /dev/null and b/pkg/bundle/zim/testdata/wikibooks_af_all_maxi_2023-06.zim differ diff --git a/pkg/bundle/zim/uncompressed_blob_reader.go b/pkg/bundle/zim/uncompressed_blob_reader.go new file mode 100644 index 0000000..5da60b7 --- /dev/null +++ b/pkg/bundle/zim/uncompressed_blob_reader.go @@ -0,0 +1,86 @@ +package zim + +import ( + "io" + "sync" + + "github.com/pkg/errors" +) + +type UncompressedBlobReader struct { + reader *Reader + blobStartOffset uint64 + blobEndOffset uint64 + blobSize int + readOffset int + + blobData []byte + loadBlobOnce sync.Once + loadBlobErr error +} + +// Size implements BlobReader. +func (r *UncompressedBlobReader) Size() (int64, error) { + return int64(r.blobEndOffset - r.blobStartOffset), nil +} + +// Close implements io.ReadCloser. +func (r *UncompressedBlobReader) Close() error { + clear(r.blobData) + return nil +} + +// Read implements io.ReadCloser. +func (r *UncompressedBlobReader) Read(p []byte) (n int, err error) { + blobData, err := r.loadBlob() + if err != nil { + return 0, errors.WithStack(err) + } + + chunkLength := len(p) + remaining := int(len(blobData) - r.readOffset) + if chunkLength > remaining { + chunkLength = remaining + } + + chunk := blobData[r.readOffset : r.readOffset+chunkLength] + r.readOffset += chunkLength + + copy(p, chunk) + + if chunkLength == remaining { + return chunkLength, io.EOF + } + + return chunkLength, nil +} + +func (r *UncompressedBlobReader) loadBlob() ([]byte, error) { + r.loadBlobOnce.Do(func() { + data := make([]byte, r.blobEndOffset-r.blobStartOffset) + err := r.reader.readRange(int64(r.blobStartOffset), data) + if err != nil { + r.loadBlobErr = errors.WithStack(err) + return + } + + r.blobData = data + }) + if r.loadBlobErr != nil { + return nil, errors.WithStack(r.loadBlobErr) + } + + return r.blobData, nil +} + +func NewUncompressedBlobReader(reader *Reader, blobStartOffset, blobEndOffset uint64, blobSize int) *UncompressedBlobReader { + return &UncompressedBlobReader{ + reader: reader, + blobStartOffset: blobStartOffset, + blobEndOffset: blobEndOffset, + blobSize: blobSize, + readOffset: 0, + } +} + +var _ BlobReader = &UncompressedBlobReader{} diff --git a/pkg/bundle/zim/util.go b/pkg/bundle/zim/util.go new file mode 100644 index 0000000..fd53347 --- /dev/null +++ b/pkg/bundle/zim/util.go @@ -0,0 +1,52 @@ +package zim + +import ( + "bytes" + "encoding/binary" + + "github.com/pkg/errors" +) + +// read a little endian uint64 +func readUint64(b []byte, order binary.ByteOrder) (uint64, error) { + var v uint64 + buf := bytes.NewBuffer(b) + if err := binary.Read(buf, order, &v); err != nil { + return 0, errors.WithStack(err) + } + + return v, nil +} + +// read a little endian uint32 +func readUint32(b []byte, order binary.ByteOrder) (uint32, error) { + var v uint32 + buf := bytes.NewBuffer(b) + if err := binary.Read(buf, order, &v); err != nil { + return 0, errors.WithStack(err) + } + + return v, nil +} + +// read a little endian uint16 +func readUint16(b []byte, order binary.ByteOrder) (uint16, error) { + var v uint16 + buf := bytes.NewBuffer(b) + if err := binary.Read(buf, order, &v); err != nil { + return 0, errors.WithStack(err) + } + + return v, nil +} + +// read a little endian uint8 +func readUint8(b []byte, order binary.ByteOrder) (uint8, error) { + var v uint8 + buf := bytes.NewBuffer(b) + if err := binary.Read(buf, order, &v); err != nil { + return 0, errors.WithStack(err) + } + + return v, nil +} diff --git a/pkg/bundle/zim/xz_blob_reader.go b/pkg/bundle/zim/xz_blob_reader.go new file mode 100644 index 0000000..420ab30 --- /dev/null +++ b/pkg/bundle/zim/xz_blob_reader.go @@ -0,0 +1,42 @@ +package zim + +import ( + "io" + + "github.com/pkg/errors" + "github.com/ulikunitz/xz" +) + +type XZBlobReader struct { + decoder *xz.Reader +} + +// Close implements io.ReadCloser. +func (r *XZBlobReader) Close() error { + return nil +} + +// Read implements io.ReadCloser. +func (r *XZBlobReader) Read(p []byte) (n int, err error) { + return r.decoder.Read(p) +} + +var _ io.ReadCloser = &XZBlobReader{} + +func NewXZBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader { + return NewCompressedBlobReader( + reader, + func(r io.Reader) (io.ReadCloser, error) { + decoder, err := xz.NewReader(r) + if err != nil { + return nil, errors.WithStack(err) + } + + return &XZBlobReader{decoder}, nil + }, + clusterStartOffset, + clusterEndOffset, + blobIndex, + blobSize, + ) +} diff --git a/pkg/bundle/zim/zstd_blob_reader.go b/pkg/bundle/zim/zstd_blob_reader.go new file mode 100644 index 0000000..ebc88cf --- /dev/null +++ b/pkg/bundle/zim/zstd_blob_reader.go @@ -0,0 +1,43 @@ +package zim + +import ( + "io" + + "github.com/klauspost/compress/zstd" + "github.com/pkg/errors" +) + +type ZstdBlobReader struct { + decoder *zstd.Decoder +} + +// Close implements io.ReadCloser. +func (r *ZstdBlobReader) Close() error { + r.decoder.Close() + return nil +} + +// Read implements io.ReadCloser. +func (r *ZstdBlobReader) Read(p []byte) (n int, err error) { + return r.decoder.Read(p) +} + +var _ io.ReadCloser = &ZstdBlobReader{} + +func NewZStdBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader { + return NewCompressedBlobReader( + reader, + func(r io.Reader) (io.ReadCloser, error) { + decoder, err := zstd.NewReader(r) + if err != nil { + return nil, errors.WithStack(err) + } + + return &ZstdBlobReader{decoder}, nil + }, + clusterStartOffset, + clusterEndOffset, + blobIndex, + blobSize, + ) +} diff --git a/pkg/bundle/zim_bundle.go b/pkg/bundle/zim_bundle.go new file mode 100644 index 0000000..8e2f90d --- /dev/null +++ b/pkg/bundle/zim_bundle.go @@ -0,0 +1,483 @@ +package bundle + +import ( + "bytes" + "context" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "golang.org/x/net/html" + + "forge.cadoles.com/arcad/edge/pkg/bundle/zim" + lru "github.com/hashicorp/golang-lru/v2" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" + "gopkg.in/yaml.v2" +) + +type ZimBundle struct { + archivePath string + + initOnce sync.Once + initErr error + + reader *zim.Reader + urlNamespaceCache *lru.Cache[string, zim.Namespace] +} + +func (b *ZimBundle) File(filename string) (io.ReadCloser, os.FileInfo, error) { + ctx := logger.With( + context.Background(), + logger.F("filename", filename), + ) + + logger.Debug(ctx, "opening file") + + switch filename { + case "manifest.yml": + return b.renderFakeManifest(ctx) + case "server/main.js": + return b.renderFakeServerMain(ctx) + case "public": + return b.renderDirectory(ctx, filename) + case "public/index.html": + return b.renderMainPage(ctx, filename) + + default: + return b.renderURL(ctx, filename) + } +} + +func (b *ZimBundle) Dir(dirname string) ([]os.FileInfo, error) { + files := make([]os.FileInfo, 0) + return files, nil +} + +func (b *ZimBundle) renderFakeManifest(ctx context.Context) (io.ReadCloser, os.FileInfo, error) { + if err := b.init(); err != nil { + return nil, nil, errors.WithStack(err) + } + + metadata, err := b.reader.Metadata() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + manifest := map[string]any{} + + manifest["version"] = "0.0.0" + + if name, exists := metadata[zim.MetadataName]; exists { + replacer := strings.NewReplacer( + "_", "", + " ", "", + ) + + manifest["id"] = strings.ToLower(replacer.Replace(name)) + ".zim.edge.app" + } else { + manifest["id"] = b.reader.UUID() + ".zim.edge.app" + } + + if title, exists := metadata[zim.MetadataTitle]; exists { + manifest["title"] = title + } else { + manifest["title"] = "Unknown" + } + + if description, exists := metadata[zim.MetadataDescription]; exists { + manifest["description"] = description + } + + favicon, err := b.reader.Favicon() + if err != nil && !errors.Is(err, zim.ErrNotFound) { + return nil, nil, errors.WithStack(err) + } + + if favicon != nil { + manifestMeta, exists := manifest["metadata"].(map[string]any) + if !exists { + manifestMeta = make(map[string]any) + manifest["metadata"] = manifestMeta + } + + paths, exists := manifestMeta["paths"].(map[string]any) + if !exists { + paths = make(map[string]any) + manifestMeta["paths"] = paths + } + + paths["icon"] = "/" + favicon.FullURL() + } + + data, err := yaml.Marshal(manifest) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + stat := &zimFileInfo{ + isDir: false, + modTime: time.Time{}, + mode: 0, + name: "manifest.yml", + size: int64(len(data)), + } + + buf := bytes.NewBuffer(data) + file := io.NopCloser(buf) + + return file, stat, nil +} + +func (b *ZimBundle) renderFakeServerMain(ctx context.Context) (io.ReadCloser, os.FileInfo, error) { + stat := &zimFileInfo{ + isDir: false, + modTime: time.Time{}, + mode: 0, + name: "server/main.js", + size: 0, + } + + buf := bytes.NewBuffer(nil) + file := io.NopCloser(buf) + + return file, stat, nil +} + +func (b *ZimBundle) renderURL(ctx context.Context, url string) (io.ReadCloser, os.FileInfo, error) { + if err := b.init(); err != nil { + return nil, nil, errors.WithStack(err) + } + + url = strings.TrimPrefix(url, "public/") + + entry, err := b.searchEntryFromURL(ctx, url) + if err != nil { + if errors.Is(err, zim.ErrNotFound) { + return nil, nil, os.ErrNotExist + } + + return nil, nil, errors.WithStack(err) + } + + logger.Debug( + ctx, "found zim entry", + logger.F("webURL", url), + logger.F("zimFullURL", entry.FullURL()), + ) + + content, err := entry.Redirect() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + contentReader, err := content.Reader() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + size, err := contentReader.Size() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + filename := filepath.Base(url) + + mimeType := content.MimeType() + if mimeType != "text/html" { + zimFile := &zimFile{ + fileInfo: &zimFileInfo{ + isDir: false, + modTime: time.Time{}, + mode: 0, + name: filename, + size: size, + }, + reader: contentReader, + } + + return zimFile, zimFile.fileInfo, nil + } + + // Read HTML file and inject Edge scripts + + data, err := io.ReadAll(contentReader) + if err != nil { + return nil, nil, err + } + + injected, err := b.injectEdgeScriptTag(data) + if err != nil { + logger.Error(ctx, "could not inject edge script", logger.E(errors.WithStack(err))) + } else { + data = injected + } + + zimFile := &zimFile{ + fileInfo: &zimFileInfo{ + isDir: false, + modTime: time.Time{}, + mode: 0, + name: filename, + size: size, + }, + reader: io.NopCloser(bytes.NewBuffer(data)), + } + + return zimFile, zimFile.fileInfo, nil +} + +func (b *ZimBundle) searchEntryFromURL(ctx context.Context, url string) (zim.Entry, error) { + ctx = logger.With(ctx, logger.F("webURL", url)) + + logger.Debug(ctx, "searching entry namespace in local cache") + + entry, err := b.reader.EntryWithFullURL(url) + if err != nil && !errors.Is(err, zim.ErrNotFound) { + return nil, errors.WithStack(err) + } + + if entry != nil { + return entry, nil + } + + contentNamespaces := []zim.Namespace{ + zim.V6NamespaceContent, + zim.V6NamespaceMetadata, + zim.V5NamespaceLayout, + zim.V5NamespaceArticle, + zim.V5NamespaceImageFile, + zim.V5NamespaceMetadata, + } + + logger.Debug( + ctx, "make educated guesses about potential url namespace", + logger.F("zimNamespaces", contentNamespaces), + ) + + for _, ns := range contentNamespaces { + logger.Debug( + ctx, "trying to access entry directly", + logger.F("zimNamespace", ns), + logger.F("zimURL", url), + ) + + entry, err := b.reader.EntryWithURL(ns, url) + if err != nil && !errors.Is(err, zim.ErrNotFound) { + return nil, errors.WithStack(err) + } + + if entry != nil { + b.urlNamespaceCache.Add(url, entry.Namespace()) + return entry, nil + } + } + + logger.Debug(ctx, "doing full entries scan") + + iterator := b.reader.Entries() + for iterator.Next() { + current := iterator.Entry() + + if current.FullURL() != url && current.URL() != url { + continue + } + + entry = current + b.urlNamespaceCache.Add(url, entry.Namespace()) + break + } + if err := iterator.Err(); err != nil { + return nil, errors.WithStack(err) + } + + if entry == nil { + return nil, errors.WithStack(zim.ErrNotFound) + } + + return entry, nil +} + +func (b *ZimBundle) renderDirectory(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) { + zimFile := &zimFile{ + fileInfo: &zimFileInfo{ + isDir: true, + modTime: time.Time{}, + mode: 0, + name: filename, + size: 0, + }, + reader: io.NopCloser(bytes.NewBuffer(nil)), + } + + return zimFile, zimFile.fileInfo, nil +} + +func (b *ZimBundle) renderMainPage(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) { + if err := b.init(); err != nil { + return nil, nil, errors.WithStack(err) + } + + main, err := b.reader.MainPage() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + return b.renderURL(ctx, main.FullURL()) +} + +func (b *ZimBundle) injectEdgeScriptTag(data []byte) ([]byte, error) { + buff := bytes.NewBuffer(data) + doc, err := html.Parse(buff) + if err != nil { + return nil, errors.WithStack(err) + } + + var f func(*html.Node) bool + f = func(n *html.Node) bool { + if n.Type == html.ElementNode && n.Data == "head" { + script := &html.Node{ + Type: html.ElementNode, + Data: "script", + Attr: []html.Attribute{ + { + Key: "src", + Val: "/edge/sdk/client.js", + }, + }, + } + + n.AppendChild(script) + + return false + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + if keepWalking := f(c); !keepWalking { + return false + } + } + + return true + } + + f(doc) + + buff.Reset() + + if err := html.Render(buff, doc); err != nil { + return nil, errors.WithStack(err) + } + + return buff.Bytes(), nil +} + +func (b *ZimBundle) init() error { + b.initOnce.Do(func() { + reader, err := zim.Open(b.archivePath) + if err != nil { + b.initErr = errors.Wrapf(err, "could not open '%v'", b.archivePath) + return + } + + b.reader = reader + + cache, err := lru.New[string, zim.Namespace](128) + if err != nil { + b.initErr = errors.Wrap(err, "could not initialize cache") + return + } + + b.urlNamespaceCache = cache + }) + if b.initErr != nil { + return errors.WithStack(b.initErr) + } + + return nil +} + +func NewZimBundle(archivePath string) *ZimBundle { + return &ZimBundle{ + archivePath: archivePath, + } +} + +type zimFile struct { + fileInfo *zimFileInfo + reader io.ReadCloser +} + +// Close implements fs.File. +func (f *zimFile) Close() error { + if err := f.reader.Close(); err != nil { + return errors.WithStack(err) + } + + return nil +} + +// Read implements fs.File. +func (f *zimFile) Read(d []byte) (int, error) { + n, err := f.reader.Read(d) + if err != nil { + if errors.Is(err, io.EOF) { + return n, err + } + + return n, errors.WithStack(err) + } + + return n, nil +} + +// Stat implements fs.File. +func (f *zimFile) Stat() (fs.FileInfo, error) { + return f.fileInfo, nil +} + +var _ fs.File = &zimFile{} + +type zimFileInfo struct { + isDir bool + modTime time.Time + mode fs.FileMode + name string + size int64 +} + +// IsDir implements fs.FileInfo. +func (i *zimFileInfo) IsDir() bool { + return i.isDir +} + +// ModTime implements fs.FileInfo. +func (i *zimFileInfo) ModTime() time.Time { + return i.modTime +} + +// Mode implements fs.FileInfo. +func (i *zimFileInfo) Mode() fs.FileMode { + return i.mode +} + +// Name implements fs.FileInfo. +func (i *zimFileInfo) Name() string { + return i.name +} + +// Size implements fs.FileInfo. +func (i *zimFileInfo) Size() int64 { + return i.size +} + +// Sys implements fs.FileInfo. +func (*zimFileInfo) Sys() any { + return nil +} + +var _ fs.FileInfo = &zimFileInfo{} diff --git a/pkg/http/html5_fileserver.go b/pkg/http/html5_fileserver.go index 56a26db..f7178af 100644 --- a/pkg/http/html5_fileserver.go +++ b/pkg/http/html5_fileserver.go @@ -27,7 +27,6 @@ func HTML5Fileserver(fs http.FileSystem) http.Handler { r.URL.Path = "/" handler.ServeHTTP(w, r) - return }