From 87a6113cc7753e9a3f13cb30cb80b87f384280d8 Mon Sep 17 00:00:00 2001 From: Thomas Miceli <27960254+thomiceli@users.noreply.github.com> Date: Thu, 4 Jan 2024 03:38:15 +0100 Subject: [PATCH] Add Gist code search (#194) --- config.yml | 6 + docs/configuration/cheat-sheet.md | 2 + go.mod | 23 ++++ go.sum | 47 ++++++++ internal/actions/actions.go | 29 ++++- internal/config/config.go | 4 + internal/db/gist.go | 123 ++++++++++++++++++++ internal/i18n/locales/en-US.yml | 9 ++ internal/index/bleve.go | 157 ++++++++++++++++++++++++++ internal/index/gist.go | 21 ++++ internal/ssh/git_ssh.go | 1 + internal/web/admin.go | 9 ++ internal/web/gist.go | 75 +++++++++++- internal/web/git_http.go | 1 + internal/web/server.go | 28 ++++- internal/web/test/gist_test.go | 3 +- internal/web/test/server.go | 14 ++- internal/web/util.go | 40 +++++++ opengist.go | 8 ++ public/main.ts | 9 ++ templates/base/base_header.html | 15 ++- templates/pages/admin_index.html | 10 +- templates/pages/all.html | 71 +----------- templates/pages/search.html | 40 +++++++ templates/partials/_gist_preview.html | 76 +++++++++++++ 25 files changed, 734 insertions(+), 87 deletions(-) create mode 100644 internal/index/bleve.go create mode 100644 internal/index/gist.go create mode 100644 templates/pages/search.html create mode 100644 templates/partials/_gist_preview.html diff --git a/config.yml b/config.yml index 5984055..168e38f 100644 --- a/config.yml +++ b/config.yml @@ -17,6 +17,12 @@ opengist-home: # Name of the SQLite database file. Default: opengist.db db-filename: opengist.db +# Enable or disable the code search index (either `true` or `false`). Default: true +index.enabled: true + +# Name of the directory where the code search index is stored. Default: opengist.index +index.dirname: opengist.index + # Default branch name used by Opengist when initializing Git repositories. # If not set, uses the Git default branch name. See https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch git.default-branch: diff --git a/docs/configuration/cheat-sheet.md b/docs/configuration/cheat-sheet.md index 7368156..e75d410 100644 --- a/docs/configuration/cheat-sheet.md +++ b/docs/configuration/cheat-sheet.md @@ -7,6 +7,8 @@ | external-url | OG_EXTERNAL_URL | none | Public URL to access to Opengist. | | opengist-home | OG_OPENGIST_HOME | home directory | Path to the directory where Opengist stores its data. | | db-filename | OG_DB_FILENAME | `opengist.db` | Name of the SQLite database file. | +| index.enabled | OG_INDEX_ENABLED | `true` | Enable or disable the code search index (`true` or `false`) | +| index.dirname | OG_INDEX_DIRNAME | `opengist.index` | Name of the directory where the code search index is stored. | | git.default-branch | OG_GIT_DEFAULT_BRANCH | none | Default branch name used by Opengist when initializing Git repositories. If not set, uses the Git default branch name. More info [here](https://git-scm.com/book/en/v2/Getting-Started-First-Time-Git-Setup#_new_default_branch) | | sqlite.journal-mode | OG_SQLITE_JOURNAL_MODE | `WAL` | Set the journal mode for SQLite. More info [here](https://www.sqlite.org/pragma.html#pragma_journal_mode) | | http.host | OG_HTTP_HOST | `0.0.0.0` | The host on which the HTTP server should bind. | diff --git a/go.mod b/go.mod index c4f2b3a..946f988 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/Kunde21/markdownfmt/v3 v3.1.0 github.com/alecthomas/chroma/v2 v2.12.0 + github.com/blevesearch/bleve/v2 v2.3.10 github.com/dustin/go-humanize v1.0.1 github.com/glebarez/go-sqlite v1.21.2 github.com/glebarez/sqlite v1.9.0 @@ -27,28 +28,50 @@ require ( ) require ( + github.com/RoaringBitmap/roaring v1.2.3 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/blevesearch/bleve_index_api v1.0.6 // indirect + github.com/blevesearch/geo v0.1.18 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/blevesearch/gtreap v0.1.1 // indirect + github.com/blevesearch/mmap-go v1.0.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect + github.com/blevesearch/segment v0.9.1 // indirect + github.com/blevesearch/snowballstem v0.9.0 // indirect + github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect + github.com/blevesearch/vellum v1.0.10 // indirect + github.com/blevesearch/zapx/v11 v11.3.10 // indirect + github.com/blevesearch/zapx/v12 v12.3.10 // indirect + github.com/blevesearch/zapx/v13 v13.3.10 // indirect + github.com/blevesearch/zapx/v14 v14.3.10 // indirect + github.com/blevesearch/zapx/v15 v15.3.13 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.10.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/net v0.15.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.12.0 // indirect diff --git a/go.sum b/go.sum index f3b20ca..4b33917 100644 --- a/go.sum +++ b/go.sum @@ -36,12 +36,48 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Kunde21/markdownfmt/v3 v3.1.0 h1:KiZu9LKs+wFFBQKhrZJrFZwtLnCCWJahL+S+E/3VnM0= github.com/Kunde21/markdownfmt/v3 v3.1.0/go.mod h1:tPXN1RTyOzJwhfHoon9wUr4HGYmWgVxSQN6VBJDkrVc= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink= github.com/alecthomas/chroma/v2 v2.2.0/go.mod h1:vf4zrexSH54oEjJ7EdB65tGNHmH3pGZmVkgTP5RHvAs= github.com/alecthomas/chroma/v2 v2.12.0 h1:Wh8qLEgMMsN7mgyG8/qIpegky2Hvzr4By6gEF7cmWgw= github.com/alecthomas/chroma/v2 v2.12.0/go.mod h1:4TQu7gdfuPjSh76j78ietmqh9LiurGF0EpseFXdKMBw= github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= +github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= +github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= +github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= +github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= +github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= +github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= +github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= +github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= +github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= +github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= +github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= +github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= +github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= +github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 h1:wMSvdj3BswqfQOXp2R1bJOAE7xIQLt2dlMQDMf836VY= github.com/chromedp/chromedp v0.9.1 h1:CC7cC5p1BeLiiS2gfNNPwp3OaUxtRMBjfiw3E3k6dFA= @@ -91,6 +127,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -119,6 +157,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -131,6 +171,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -177,6 +218,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede h1:YrgBGwxMRK0Vq0WSCWFaZUnTsrA/PZE/xs1QZh+/edg= +github.com/json-iterator/go v0.0.0-20171115153421-f7279a603ede/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -212,6 +255,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mrjones/oauth v0.0.0-20180629183705-f4e24b6d100c/go.mod h1:skjdDftzkFALcuGzYSklqYd8gvat6F1gZJ4YPVbkZpM= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -254,6 +299,8 @@ github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc h1:+ github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc/go.mod h1:ovIvrum6DQJA4QsJSovrkC4saKHQVs7TvcaeO8AIl5I= go.abhg.dev/goldmark/mermaid v0.5.0 h1:mDkykpSPJ+5wCQ8bSXgzJ2KQskjXkI5Ndxz7JYDHW38= go.abhg.dev/goldmark/mermaid v0.5.0/go.mod h1:OCyk2o85TX2drWHH+HRy6bih2yZlUwbbv/R1MMh1YLs= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/internal/actions/actions.go b/internal/actions/actions.go index 93dc28c..629cf3b 100644 --- a/internal/actions/actions.go +++ b/internal/actions/actions.go @@ -5,6 +5,7 @@ import ( "github.com/thomiceli/opengist/internal/config" "github.com/thomiceli/opengist/internal/db" "github.com/thomiceli/opengist/internal/git" + "github.com/thomiceli/opengist/internal/index" "os" "path/filepath" "strings" @@ -21,6 +22,7 @@ const ( GitGcRepos = iota SyncGistPreviews = iota ResetHooks = iota + IndexGists = iota ) var ( @@ -69,6 +71,8 @@ func Run(actionType int) { functionToRun = syncGistPreviews case ResetHooks: functionToRun = resetHooks + case IndexGists: + functionToRun = indexGists default: panic("unhandled default case") } @@ -88,7 +92,6 @@ func syncReposFromFS() { if _, err := os.Stat(git.RepositoryPath(gist.User.Username, gist.Uuid)); err != nil && !os.IsExist(err) { if err2 := gist.Delete(); err2 != nil { log.Error().Err(err2).Msgf("Cannot delete gist %d", gist.ID) - return } } } @@ -109,7 +112,6 @@ func syncReposFromDB() { if gist.ID == 0 { if err := git.DeleteRepository(path[len(path)-2], path[len(path)-1]); err != nil { log.Error().Err(err).Msgf("Cannot delete repository %s/%s", path[len(path)-2], path[len(path)-1]) - return } } } @@ -133,7 +135,6 @@ func syncGistPreviews() { for _, gist := range gists { if err = gist.UpdatePreviewAndCount(false); err != nil { log.Error().Err(err).Msgf("Cannot update preview and count for gist %d", gist.ID) - return } } } @@ -150,7 +151,27 @@ func resetHooks() { path := strings.Split(e, string(os.PathSeparator)) if err := git.CreateDotGitFiles(path[len(path)-2], path[len(path)-1]); err != nil { log.Error().Err(err).Msgf("Cannot reset hooks for repository %s/%s", path[len(path)-2], path[len(path)-1]) - return + } + } +} + +func indexGists() { + log.Info().Msg("Indexing all Gists...") + gists, err := db.GetAllGistsRows() + if err != nil { + log.Error().Err(err).Msg("Cannot get gists") + return + } + + for _, gist := range gists { + log.Info().Msgf("Indexing gist %d", gist.ID) + indexedGist, err := gist.ToIndexedGist() + if err != nil { + log.Error().Err(err).Msgf("Cannot convert gist %d to indexed gist", gist.ID) + continue + } + if err = index.AddInIndex(indexedGist); err != nil { + log.Error().Err(err).Msgf("Cannot index gist %d", gist.ID) } } } diff --git a/internal/config/config.go b/internal/config/config.go index d575bc9..e7e53ea 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -28,6 +28,8 @@ type config struct { ExternalUrl string `yaml:"external-url" env:"OG_EXTERNAL_URL"` OpengistHome string `yaml:"opengist-home" env:"OG_OPENGIST_HOME"` DBFilename string `yaml:"db-filename" env:"OG_DB_FILENAME"` + IndexEnabled bool `yaml:"index.enabled" env:"OG_INDEX_ENABLED"` + IndexDirname string `yaml:"index.dirname" env:"OG_INDEX_DIRNAME"` GitDefaultBranch string `yaml:"git.default-branch" env:"OG_GIT_DEFAULT_BRANCH"` @@ -66,6 +68,8 @@ func configWithDefaults() (*config, error) { c.LogOutput = "stdout,file" c.OpengistHome = "" c.DBFilename = "opengist.db" + c.IndexEnabled = true + c.IndexDirname = "opengist.index" c.SqliteJournalMode = "WAL" diff --git a/internal/db/gist.go b/internal/db/gist.go index 73a89ab..4b526c8 100644 --- a/internal/db/gist.go +++ b/internal/db/gist.go @@ -2,9 +2,14 @@ package db import ( "fmt" + "github.com/alecthomas/chroma/v2" + "github.com/alecthomas/chroma/v2/lexers" "github.com/dustin/go-humanize" "github.com/labstack/echo/v4" + "github.com/rs/zerolog/log" + "github.com/thomiceli/opengist/internal/index" "os/exec" + "path/filepath" "strings" "time" @@ -213,6 +218,25 @@ func GetAllGistsRows() ([]*Gist, error) { return gists, err } +func GetAllGistsVisibleByUser(userId uint) ([]uint, error) { + var gists []uint + + err := db.Table("gists"). + Where("gists.private = 0 or gists.user_id = ?", userId). + Pluck("gists.id", &gists).Error + + return gists, err +} + +func GetAllGistsByIds(ids []uint) ([]*Gist, error) { + var gists []*Gist + err := db.Preload("User").Preload("Forked.User"). + Where("id in ?", ids). + Find(&gists).Error + + return gists, err +} + func (gist *Gist) Create() error { // avoids foreign key constraint error because the default value in the struct is 0 return db.Omit("forked_id").Create(&gist).Error @@ -361,6 +385,10 @@ func (gist *Gist) File(revision string, filename string, truncate bool) (*git.Fi }, err } +func (gist *Gist) FileNames(revision string) ([]string, error) { + return git.GetFilesOfRepository(gist.User.Username, gist.Uuid, revision) +} + func (gist *Gist) Log(skip int) ([]*git.Commit, error) { return git.GetLog(gist.User.Username, gist.Uuid, skip) } @@ -475,6 +503,30 @@ func (gist *Gist) Identifier() string { return gist.Uuid } +func (gist *Gist) GetLanguagesFromFiles() ([]string, error) { + files, err := gist.Files("HEAD", true) + if err != nil { + return nil, err + } + + languages := make([]string, 0, len(files)) + for _, file := range files { + var lexer chroma.Lexer + if lexer = lexers.Get(file.Filename); lexer == nil { + lexer = lexers.Fallback + } + + fileType := lexer.Config().Name + if lexer.Config().Name == "fallback" || lexer.Config().Name == "plaintext" { + fileType = "Text" + } + + languages = append(languages, fileType) + } + + return languages, nil +} + // -- DTO -- // type GistDTO struct { @@ -507,3 +559,74 @@ func (dto *GistDTO) ToExistingGist(gist *Gist) *Gist { gist.URL = dto.URL return gist } + +// -- Index -- // + +func (gist *Gist) ToIndexedGist() (*index.Gist, error) { + files, err := gist.Files("HEAD", true) + if err != nil { + return nil, err + } + + exts := make([]string, 0, len(files)) + wholeContent := "" + for _, file := range files { + wholeContent += file.Content + exts = append(exts, filepath.Ext(file.Filename)) + } + + fileNames, err := gist.FileNames("HEAD") + if err != nil { + return nil, err + } + + langs, err := gist.GetLanguagesFromFiles() + if err != nil { + return nil, err + } + + indexedGist := &index.Gist{ + GistID: gist.ID, + Username: gist.User.Username, + Title: gist.Title, + Content: wholeContent, + Filenames: fileNames, + Extensions: exts, + Languages: langs, + CreatedAt: gist.CreatedAt, + UpdatedAt: gist.UpdatedAt, + } + + return indexedGist, nil +} + +func (gist *Gist) AddInIndex() { + if !index.Enabled() { + return + } + + go func() { + indexedGist, err := gist.ToIndexedGist() + if err != nil { + log.Error().Err(err).Msgf("Cannot convert gist %d to indexed gist", gist.ID) + return + } + err = index.AddInIndex(indexedGist) + if err != nil { + log.Error().Err(err).Msgf("Error adding gist %d to index", gist.ID) + } + }() +} + +func (gist *Gist) RemoveFromIndex() { + if !index.Enabled() { + return + } + + go func() { + err := index.RemoveFromIndex(gist.ID) + if err != nil { + log.Error().Err(err).Msgf("Error remove gist %d from index", gist.ID) + } + }() +} diff --git a/internal/i18n/locales/en-US.yml b/internal/i18n/locales/en-US.yml index 0c136bb..092dec1 100644 --- a/internal/i18n/locales/en-US.yml +++ b/internal/i18n/locales/en-US.yml @@ -68,6 +68,14 @@ gist.list.files: files gist.list.last-active: Last active gist.list.no-gists: No gists +gist.search.found: gists found +gist.search.no-results: No gists found +gist.search.help.user: gists created by user +gist.search.help.title: gists with given title +gist.search.help.filename: gists having files with given name +gist.search.help.extension: gists having files with given extension +gist.search.help.language: gists having files with given language + gist.forks: Forks gist.forks.view: View fork gist.forks.no: No public forks @@ -164,6 +172,7 @@ admin.actions.sync-db: Synchronize gists from database admin.actions.git-gc: Garbage collect all git repositories admin.actions.sync-previews: Synchronize all gists previews admin.actions.reset-hooks: Reset Git server hooks for all repositories +admin.actions.index-gists: Index all gists admin.id: ID admin.user: User admin.delete: Delete diff --git a/internal/index/bleve.go b/internal/index/bleve.go new file mode 100644 index 0000000..9abcc29 --- /dev/null +++ b/internal/index/bleve.go @@ -0,0 +1,157 @@ +package index + +import ( + "errors" + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + "github.com/blevesearch/bleve/v2/analysis/token/camelcase" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/search/query" + "github.com/thomiceli/opengist/internal/config" + "strconv" +) + +var bleveIndex bleve.Index + +func Enabled() bool { + return config.C.IndexEnabled +} + +func Open(indexFilename string) error { + var err error + bleveIndex, err = bleve.Open(indexFilename) + if err == nil { + return nil + } + + if !errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { + return err + } + + docMapping := bleve.NewDocumentMapping() + docMapping.AddFieldMappingsAt("GistID", bleve.NewNumericFieldMapping()) + docMapping.AddFieldMappingsAt("Content", bleve.NewTextFieldMapping()) + + mapping := bleve.NewIndexMapping() + + if err = mapping.AddCustomTokenFilter("unicodeNormalize", map[string]any{ + "type": unicodenorm.Name, + "form": unicodenorm.NFC, + }); err != nil { + return err + } + + if err = mapping.AddCustomAnalyzer("gistAnalyser", map[string]interface{}{ + "type": custom.Name, + "char_filters": []string{}, + "tokenizer": unicode.Name, + "token_filters": []string{"unicodeNormalize", camelcase.Name, lowercase.Name}, + }); err != nil { + return err + } + + docMapping.DefaultAnalyzer = "gistAnalyser" + + bleveIndex, err = bleve.New(indexFilename, mapping) + + return err +} + +func Close() error { + return bleveIndex.Close() +} + +func AddInIndex(gist *Gist) error { + if !Enabled() { + return nil + } + + if gist == nil { + return errors.New("failed to add nil gist to index") + } + return bleveIndex.Index(strconv.Itoa(int(gist.GistID)), gist) +} + +func RemoveFromIndex(gistID uint) error { + if !Enabled() { + return nil + } + + return bleveIndex.Delete(strconv.Itoa(int(gistID))) +} + +func SearchGists(queryStr string, queryMetadata SearchGistMetadata, gistsIds []uint, page int) ([]uint, uint64, map[string]int, error) { + if !Enabled() { + return nil, 0, nil, nil + } + + var err error + var indexerQuery query.Query + if queryStr != "" { + contentQuery := bleve.NewMatchPhraseQuery(queryStr) + contentQuery.FieldVal = "Content" + indexerQuery = contentQuery + } else { + contentQuery := bleve.NewMatchAllQuery() + indexerQuery = contentQuery + } + + if len(gistsIds) > 0 { + repoQueries := make([]query.Query, 0, len(gistsIds)) + + truee := true + for _, id := range gistsIds { + f := float64(id) + qq := bleve.NewNumericRangeInclusiveQuery(&f, &f, &truee, &truee) + qq.SetField("GistID") + repoQueries = append(repoQueries, qq) + } + + indexerQuery = bleve.NewConjunctionQuery(bleve.NewDisjunctionQuery(repoQueries...), indexerQuery) + } + + addQuery := func(field, value string) { + if value != "" && value != "." { + q := bleve.NewMatchPhraseQuery(value) + q.FieldVal = field + indexerQuery = bleve.NewConjunctionQuery(indexerQuery, q) + } + } + + addQuery("Username", queryMetadata.Username) + addQuery("Title", queryMetadata.Title) + addQuery("Extensions", "."+queryMetadata.Extension) + addQuery("Filenames", queryMetadata.Filename) + addQuery("Languages", queryMetadata.Language) + + languageFacet := bleve.NewFacetRequest("Languages", 10) + + perPage := 10 + offset := (page - 1) * perPage + + s := bleve.NewSearchRequestOptions(indexerQuery, perPage, offset, false) + s.AddFacet("languageFacet", languageFacet) + s.Fields = []string{"GistID"} + s.IncludeLocations = false + + results, err := bleveIndex.Search(s) + if err != nil { + return nil, 0, nil, err + } + + gistIds := make([]uint, 0, len(results.Hits)) + for _, hit := range results.Hits { + gistIds = append(gistIds, uint(hit.Fields["GistID"].(float64))) + } + + languageCounts := make(map[string]int) + if facets, found := results.Facets["languageFacet"]; found { + for _, term := range facets.Terms.Terms() { + languageCounts[term.Term] = term.Count + } + } + + return gistIds, results.Total, languageCounts, nil +} diff --git a/internal/index/gist.go b/internal/index/gist.go new file mode 100644 index 0000000..88b62e8 --- /dev/null +++ b/internal/index/gist.go @@ -0,0 +1,21 @@ +package index + +type Gist struct { + GistID uint + Username string + Title string + Content string + Filenames []string + Extensions []string + Languages []string + CreatedAt int64 + UpdatedAt int64 +} + +type SearchGistMetadata struct { + Username string + Title string + Filename string + Extension string + Language string +} diff --git a/internal/ssh/git_ssh.go b/internal/ssh/git_ssh.go index 9c3f544..f6154c8 100644 --- a/internal/ssh/git_ssh.go +++ b/internal/ssh/git_ssh.go @@ -95,6 +95,7 @@ func runGitCommand(ch ssh.Channel, gitCmd string, key string, ip string) error { if verb == "receive-pack" { _ = gist.SetLastActiveNow() _ = gist.UpdatePreviewAndCount(false) + gist.AddInIndex() } return nil diff --git a/internal/web/admin.go b/internal/web/admin.go index cfeb45d..a438510 100644 --- a/internal/web/admin.go +++ b/internal/web/admin.go @@ -46,6 +46,7 @@ func adminIndex(ctx echo.Context) error { setData(ctx, "gitGcRepos", actions.IsRunning(actions.GitGcRepos)) setData(ctx, "syncGistPreviews", actions.IsRunning(actions.SyncGistPreviews)) setData(ctx, "resetHooks", actions.IsRunning(actions.ResetHooks)) + setData(ctx, "indexGists", actions.IsRunning(actions.IndexGists)) return html(ctx, "admin_index.html") } @@ -116,6 +117,8 @@ func adminGistDelete(ctx echo.Context) error { return errorRes(500, "Cannot delete this gist", err) } + gist.RemoveFromIndex() + addFlash(ctx, "Gist has been deleted", "success") return redirect(ctx, "/admin-panel/gists") } @@ -150,6 +153,12 @@ func adminResetHooks(ctx echo.Context) error { return redirect(ctx, "/admin-panel") } +func adminIndexGists(ctx echo.Context) error { + addFlash(ctx, "Indexing all gists...", "success") + go actions.Run(actions.IndexGists) + return redirect(ctx, "/admin-panel") +} + func adminConfig(ctx echo.Context) error { setData(ctx, "title", "Configuration") setData(ctx, "htmlTitle", "Configuration - Admin panel") diff --git a/internal/web/gist.go b/internal/web/gist.go index beb51d0..47c36e4 100644 --- a/internal/web/gist.go +++ b/internal/web/gist.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/rs/zerolog/log" "github.com/thomiceli/opengist/internal/git" + "github.com/thomiceli/opengist/internal/index" "github.com/thomiceli/opengist/internal/render" "html/template" "net/url" @@ -252,20 +253,20 @@ func allGists(ctx echo.Context) error { } } - renderedFiles := make([]*render.RenderedGist, 0, len(gists)) + renderedGists := make([]*render.RenderedGist, 0, len(gists)) for _, gist := range gists { rendered, err := render.HighlightGistPreview(gist) if err != nil { - log.Warn().Err(err).Msg("Error rendering gist preview for " + gist.Identifier() + " - " + gist.PreviewFilename) + log.Error().Err(err).Msg("Error rendering gist preview for " + gist.Identifier() + " - " + gist.PreviewFilename) } - renderedFiles = append(renderedFiles, &rendered) + renderedGists = append(renderedGists, &rendered) } if err != nil { return errorRes(500, "Error fetching gists", err) } - if err = paginate(ctx, renderedFiles, pageInt, 10, "gists", fromUserStr, 2, "&sort="+sort+"&order="+order); err != nil { + if err = paginate(ctx, renderedGists, pageInt, 10, "gists", fromUserStr, 2, "&sort="+sort+"&order="+order); err != nil { return errorRes(404, "Page not found", nil) } @@ -273,6 +274,69 @@ func allGists(ctx echo.Context) error { return html(ctx, "all.html") } +func search(ctx echo.Context) error { + var err error + + content, meta := parseSearchQueryStr(ctx.QueryParam("q")) + pageInt := getPage(ctx) + + var currentUserId uint + userLogged := getUserLogged(ctx) + if userLogged != nil { + currentUserId = userLogged.ID + } else { + currentUserId = 0 + } + + var visibleGistsIds []uint + visibleGistsIds, err = db.GetAllGistsVisibleByUser(currentUserId) + if err != nil { + return errorRes(500, "Error fetching gists", err) + } + + gistsIds, nbHits, langs, err := index.SearchGists(content, index.SearchGistMetadata{ + Username: meta["user"], + Title: meta["title"], + Filename: meta["filename"], + Extension: meta["extension"], + Language: meta["language"], + }, visibleGistsIds, pageInt) + if err != nil { + return errorRes(500, "Error searching gists", err) + } + + gists, err := db.GetAllGistsByIds(gistsIds) + if err != nil { + return errorRes(500, "Error fetching gists", err) + } + + renderedGists := make([]*render.RenderedGist, 0, len(gists)) + for _, gist := range gists { + rendered, err := render.HighlightGistPreview(gist) + if err != nil { + log.Error().Err(err).Msg("Error rendering gist preview for " + gist.Identifier() + " - " + gist.PreviewFilename) + } + renderedGists = append(renderedGists, &rendered) + } + + if pageInt > 1 && len(renderedGists) != 0 { + setData(ctx, "prevPage", pageInt-1) + } + if 10*pageInt < int(nbHits) { + setData(ctx, "nextPage", pageInt+1) + } + setData(ctx, "prevLabel", tr(ctx, "pagination.previous")) + setData(ctx, "nextLabel", tr(ctx, "pagination.next")) + setData(ctx, "urlPage", "search") + setData(ctx, "urlParams", template.URL("&q="+ctx.QueryParam("q"))) + setData(ctx, "htmlTitle", "Search results") + setData(ctx, "nbHits", nbHits) + setData(ctx, "gists", renderedGists) + setData(ctx, "langs", langs) + setData(ctx, "searchQuery", ctx.QueryParam("q")) + return html(ctx, "search.html") +} + func gistIndex(ctx echo.Context) error { if getData(ctx, "gistpage") == "js" { return gistJs(ctx) @@ -545,6 +609,8 @@ func processCreate(ctx echo.Context) error { } } + gist.AddInIndex() + return redirect(ctx, "/"+user.Username+"/"+gist.Identifier()) } @@ -566,6 +632,7 @@ func deleteGist(ctx echo.Context) error { if err := gist.Delete(); err != nil { return errorRes(500, "Error deleting this gist", err) } + gist.RemoveFromIndex() addFlash(ctx, "Gist has been deleted", "success") return redirect(ctx, "/") diff --git a/internal/web/git_http.go b/internal/web/git_http.go index 7363dc6..40c0f78 100644 --- a/internal/web/git_http.go +++ b/internal/web/git_http.go @@ -218,6 +218,7 @@ func pack(ctx echo.Context, serviceType string) error { _ = gist.SetLastActiveNow() _ = gist.UpdatePreviewAndCount(false) + gist.AddInIndex() } return nil } diff --git a/internal/web/server.go b/internal/web/server.go index 71af827..45c9185 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -3,7 +3,9 @@ package web import ( "context" "encoding/json" + "errors" "fmt" + "github.com/thomiceli/opengist/internal/index" htmlpkg "html" "html/template" "io" @@ -115,6 +117,22 @@ var ( "safe": func(s string) template.HTML { return template.HTML(s) }, + "dict": func(values ...interface{}) (map[string]interface{}, error) { + if len(values)%2 != 0 { + return nil, errors.New("invalid dict call") + } + dict := make(map[string]interface{}) + for i := 0; i < len(values); i += 2 { + key, ok := values[i].(string) + if !ok { + return nil, errors.New("dict keys must be strings") + } + dict[key] = values[i+1] + } + return dict, nil + }, + "addMetadataToSearchQuery": addMetadataToSearchQuery, + "indexEnabled": index.Enabled, } ) @@ -223,7 +241,6 @@ func NewServer(isDev bool) *Server { g1.DELETE("/settings/ssh-keys/:id", sshKeysDelete, logged) g1.PUT("/settings/password", passwordProcess, logged) g1.PUT("/settings/username", usernameProcess, logged) - g2 := g1.Group("/admin-panel") { g2.Use(adminPermission) @@ -237,6 +254,7 @@ func NewServer(isDev bool) *Server { g2.POST("/gc-repos", adminGcRepos) g2.POST("/sync-previews", adminSyncGistPreviews) g2.POST("/reset-hooks", adminResetHooks) + g2.POST("/index-gists", adminIndexGists) g2.GET("/configuration", adminConfig) g2.PUT("/set-config", adminSetConfig) } @@ -246,7 +264,13 @@ func NewServer(isDev bool) *Server { } g1.GET("/all", allGists, checkRequireLogin) - g1.GET("/search", allGists, checkRequireLogin) + + if index.Enabled() { + g1.GET("/search", search, checkRequireLogin) + } else { + g1.GET("/search", allGists, checkRequireLogin) + } + g1.GET("/:user", allGists, checkRequireLogin) g1.GET("/:user/liked", allGists, checkRequireLogin) g1.GET("/:user/forked", allGists, checkRequireLogin) diff --git a/internal/web/test/gist_test.go b/internal/web/test/gist_test.go index 6462984..a04f906 100644 --- a/internal/web/test/gist_test.go +++ b/internal/web/test/gist_test.go @@ -1,11 +1,10 @@ package test import ( - "testing" - "github.com/stretchr/testify/require" "github.com/thomiceli/opengist/internal/db" "github.com/thomiceli/opengist/internal/git" + "testing" ) func TestGists(t *testing.T) { diff --git a/internal/web/test/server.go b/internal/web/test/server.go index a6853c1..d951426 100644 --- a/internal/web/test/server.go +++ b/internal/web/test/server.go @@ -133,14 +133,13 @@ func setup(t *testing.T) { git.ReposDirectory = path.Join("tests") + config.C.IndexEnabled = false + config.C.LogLevel = "debug" config.InitLog() homePath := config.GetHomeDir() log.Info().Msg("Data directory: " + homePath) - err = os.MkdirAll(filepath.Join(homePath, "repos"), 0755) - require.NoError(t, err, "Could not create repos directory") - err = os.MkdirAll(filepath.Join(homePath, "tmp", "repos"), 0755) require.NoError(t, err, "Could not create tmp repos directory") @@ -149,6 +148,9 @@ func setup(t *testing.T) { err = memdb.Setup() require.NoError(t, err, "Could not initialize in memory database") + + // err = index.Open(filepath.Join(homePath, "testsindex", "opengist.index")) + // require.NoError(t, err, "Could not open index") } func teardown(t *testing.T, s *testServer) { @@ -159,4 +161,10 @@ func teardown(t *testing.T, s *testServer) { err = os.RemoveAll(path.Join(config.C.OpengistHome, "tests")) require.NoError(t, err, "Could not remove repos directory") + + // err = os.RemoveAll(path.Join(config.C.OpengistHome, "testsindex")) + // require.NoError(t, err, "Could not remove repos directory") + + // err = index.Close() + // require.NoError(t, err, "Could not close index") } diff --git a/internal/web/util.go b/internal/web/util.go index 5c86455..5547581 100644 --- a/internal/web/util.go +++ b/internal/web/util.go @@ -248,6 +248,46 @@ func tr(ctx echo.Context, key string) template.HTML { return l.Tr(key) } +func parseSearchQueryStr(query string) (string, map[string]string) { + words := strings.Fields(query) + metadata := make(map[string]string) + var contentBuilder strings.Builder + + for _, word := range words { + if strings.Contains(word, ":") { + keyValue := strings.SplitN(word, ":", 2) + if len(keyValue) == 2 { + key := keyValue[0] + value := keyValue[1] + metadata[key] = value + } + } else { + contentBuilder.WriteString(word + " ") + } + } + + content := strings.TrimSpace(contentBuilder.String()) + return content, metadata +} + +func addMetadataToSearchQuery(input, key, value string) string { + content, metadata := parseSearchQueryStr(input) + + metadata[key] = value + + var resultBuilder strings.Builder + resultBuilder.WriteString(content) + + for k, v := range metadata { + resultBuilder.WriteString(" ") + resultBuilder.WriteString(k) + resultBuilder.WriteString(":") + resultBuilder.WriteString(v) + } + + return strings.TrimSpace(resultBuilder.String()) +} + type Argon2ID struct { format string version int diff --git a/opengist.go b/opengist.go index 3b665b6..ea36c30 100644 --- a/opengist.go +++ b/opengist.go @@ -7,6 +7,7 @@ import ( "github.com/thomiceli/opengist/internal/config" "github.com/thomiceli/opengist/internal/db" "github.com/thomiceli/opengist/internal/git" + "github.com/thomiceli/opengist/internal/index" "github.com/thomiceli/opengist/internal/memdb" "github.com/thomiceli/opengist/internal/ssh" "github.com/thomiceli/opengist/internal/web" @@ -59,6 +60,13 @@ func initialize() { if err := memdb.Setup(); err != nil { log.Fatal().Err(err).Msg("Failed to initialize in memory database") } + + if config.C.IndexEnabled { + log.Info().Msg("Index directory: " + filepath.Join(homePath, config.C.IndexDirname)) + if err := index.Open(filepath.Join(homePath, config.C.IndexDirname)); err != nil { + log.Fatal().Err(err).Msg("Failed to open index") + } + } } func main() { diff --git a/public/main.ts b/public/main.ts index b645c0b..a33f78e 100644 --- a/public/main.ts +++ b/public/main.ts @@ -167,4 +167,13 @@ document.addEventListener('DOMContentLoaded', () => { } }); } + + const searchinput = document.getElementById('search') as HTMLInputElement; + searchinput.addEventListener('focusin', () => { + document.getElementById('search-help').classList.remove('hidden'); + }) + + searchinput.addEventListener('focusout', (e) => { + document.getElementById('search-help').classList.add('hidden'); + }) }); diff --git a/templates/base/base_header.html b/templates/base/base_header.html index 565f62a..bb7883a 100644 --- a/templates/base/base_header.html +++ b/templates/base/base_header.html @@ -89,9 +89,22 @@
- +
+ {{if indexEnabled}} + + {{end}} diff --git a/templates/pages/admin_index.html b/templates/pages/admin_index.html index e4970bc..83778d6 100644 --- a/templates/pages/admin_index.html +++ b/templates/pages/admin_index.html @@ -74,18 +74,24 @@ {{ .locale.Tr "admin.actions.git-gc" }} -
+ {{ .csrfHtml }}
-
+ {{ .csrfHtml }}
+
+ {{ .csrfHtml }} + +
diff --git a/templates/pages/all.html b/templates/pages/all.html index 461205d..38e68d4 100644 --- a/templates/pages/all.html +++ b/templates/pages/all.html @@ -108,75 +108,8 @@
{{ if ne (len .gists) 0 }} {{ range $gist := .gists }} -
-
-
- - {{ $gist.User.Username }}'s Avatar - -
-
-
-

- {{ $gist.User.Username }} / {{ $gist.Title }} -

-
-
- - - - {{ $gist.NbLikes }} {{ $.locale.Tr "gist.list.likes" }} -
-
- - - - {{ $gist.NbForks }} {{ $.locale.Tr "gist.list.forks" }} -
-
- - - - {{ $gist.NbFiles }} {{ $.locale.Tr "gist.list.files" }} -
-
- -
-
{{ $.locale.Tr "gist.list.last-active" }} {{ $gist.UpdatedAt }} - {{ if $gist.Forked }} • {{ $.locale.Tr "gist.list.forked-from" }} {{ $gist.Forked.User.Username }}/{{ $gist.Forked.Title }} {{ end }} - {{ if $gist.Private }} • {{ visibilityStr $gist.Private false }} {{ end }}
-
{{ $gist.Description }}
-
-
- -
-
- {{ if $gist.PreviewFilename }} - {{ if isMarkdown $gist.PreviewFilename }} -
{{ $gist.HTML | safe }}
- {{ else }} - - - {{ $ii := "1" }} - {{ $i := toInt $ii }} - {{ range $line := $gist.Lines }} - - - - - - {{ $i = inc $i }} - {{ end }} - -
{{$i}}{{ $line | safe }}
- {{ end }} - {{ else }} -

{{ $.locale.Tr "gist.no-content" }}

- {{ end }} -
-
- -
+ {{ $nest := dict "gist" $gist "C" $.c "locale" $.locale "DisableGravatar" $.DisableGravatar "searchQuery" $.searchQuery }} + {{ template "_gist_preview" $nest }} {{ end }} {{ template "pagination" . }} diff --git a/templates/pages/search.html b/templates/pages/search.html new file mode 100644 index 0000000..095a110 --- /dev/null +++ b/templates/pages/search.html @@ -0,0 +1,40 @@ +{{ template "header" .}} +
+
+
+
+

{{ .nbHits }} {{ .locale.Tr "gist.search.found" }}

+
+
+
+
+ {{ if ne (len .gists) 0 }} +
+
+
+ {{ range $lang, $count := .langs }} + + {{ $lang }} ({{ $count }}) + + {{end}} +
+
+
+ {{ range $gist := .gists }} + {{ $nest := dict "gist" $gist "C" $.c "locale" $.locale "DisableGravatar" $.DisableGravatar }} + {{ template "_gist_preview" $nest }} + {{ end }} +
+
+ {{ template "pagination" . }} + {{ else }} +
+ + + +

{{ .locale.Tr "gist.search.no-results" }}

+
+ {{ end }} +
+
+{{ template "footer" .}} diff --git a/templates/partials/_gist_preview.html b/templates/partials/_gist_preview.html new file mode 100644 index 0000000..f4956de --- /dev/null +++ b/templates/partials/_gist_preview.html @@ -0,0 +1,76 @@ +{{ define "_gist_preview" }} + + +
+
+
+ + {{ .gist.User.Username }}'s Avatar + +
+
+
+

+ {{ .gist.User.Username }} / {{ .gist.Title }} +

+
+
+ + + + {{ .gist.NbLikes }} {{ .locale.Tr "gist.list.likes" }} +
+
+ + + + {{ .gist.NbForks }} {{ .locale.Tr "gist.list.forks" }} +
+
+ + + + {{ .gist.NbFiles }} {{ .locale.Tr "gist.list.files" }} +
+
+ +
+
{{ .locale.Tr "gist.list.last-active" }} {{ .gist.UpdatedAt }} + {{ if .gist.Forked }} • {{ .locale.Tr "gist.list.forked-from" }} {{ .gist.Forked.User.Username }}/{{ .gist.Forked.Title }} {{ end }} + {{ if .gist.Private }} • {{ visibilityStr .gist.Private false }} {{ end }}
+
{{ .gist.Description }}
+
+
+ +
+
+ {{ if .gist.PreviewFilename }} + {{ if isMarkdown .gist.PreviewFilename }} +
{{ .gist.HTML | safe }}
+ {{ else }} + + + {{ $ii := "1" }} + {{ $i := toInt $ii }} + {{ range $line := .gist.Lines }} + + + + + + {{ $i = inc $i }} + {{ end }} + +
{{$i}}{{ $line | safe }}
+ {{ end }} + {{ else }} +

{{ .locale.Tr "gist.no-content" }}

+ {{ end }} +
+
+ +
+ + +{{ end }} +