diff --git a/.env.docker b/.env.docker index cbf87f3..d6cb7c3 100644 --- a/.env.docker +++ b/.env.docker @@ -66,7 +66,7 @@ MEMCACHED_HOST=127.0.0.1 VITE_APP_NAME="${APP_NAME}" VITE_APP_URL="${APP_URL}" -DUSK_DRIVER_URL="http://undetected-chromedriver:4444" +DUSK_DRIVER_URL="http://selenium-hub:4444" DUSK_START_MAXIMIZED=true # Reverb (broadcasting) diff --git a/.env.example b/.env.example index 5129478..c45b066 100644 --- a/.env.example +++ b/.env.example @@ -89,3 +89,7 @@ LLM_CHAT_MODEL="deepseek-r1:8b" LLM_CHAT_MODEL_THINK=true LLM_VISION_MODEL="llava:7b" LLM_VISION_MODEL_THINK=false +# AI TRANSCRIPTION +TRANSC_API_HOST_URL="https://speaches.example.com/v1" +TRANSC_API_TOKEN="" +TRANSC_TRANSCRIPTION_MODEL="Infomaniak-AI/faster-whisper-large-v3-turbo" diff --git a/.gitea/workflows/pushImage.yaml b/.gitea/workflows/pushImage.yaml index ec1a0fd..922c87a 100644 --- a/.gitea/workflows/pushImage.yaml +++ b/.gitea/workflows/pushImage.yaml @@ -26,6 +26,7 @@ jobs: run: composer install --prefer-dist --no-progress --no-suggest --no-interaction - name: Run PHPUnit tests run: vendor/bin/phpunit --configuration phpunit.xml + build-image: needs: phpunit-tests if: success() && github.ref == 'refs/heads/main' diff --git a/README.md b/README.md index ba3e4bb..04ec1c7 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,4 @@ Datboi controlling a browser for you; He's doing scheduled repetitive tasks for ### Links - [novnc](http://localhost:7900/?password=secret&autoconnect=true&scale=local) : See the browser being controlled live. +- [selenium Hub](http://localhost:4444/ui/) diff --git a/compose.prod.yaml b/compose.prod.yaml index 4209d12..655eadb 100644 --- a/compose.prod.yaml +++ b/compose.prod.yaml @@ -9,19 +9,19 @@ services: - browserScreenshots:/app/Browser/screenshots - browserSource:/app/Browser/source depends_on: - db: - condition: service_healthy - undetected-chromedriver: + # db: + # condition: service_healthy + selenium-hub: condition: service_healthy db: - image: mysql:5.7 + image: mariadb:latest restart: unless-stopped - healthcheck: - test: ["CMD", 'mysqladmin', 'ping', '-h', 'db', '-u', '${DB_USERNAME}', '-p${DB_PASSWORD}' ] - interval: 2s - timeout: 20s - retries: 10 + # healthcheck: + # test: ["CMD", 'mysqladmin', 'ping', '-h', 'db', '-u', '${DB_USERNAME}', '-p${DB_PASSWORD}' ] + # interval: 2s + # timeout: 20s + # retries: 10 environment: MYSQL_ROOT_PASSWORD: ${DB_PASSWORD} MYSQL_DATABASE: ${DB_DATABASE} @@ -30,23 +30,51 @@ services: volumes: - dbdata:/var/lib/mysql - undetected-chromedriver: - image: git.matthiasg.dev/ninluc/selenium/standalone-uc:latest + undetected-chromedriver-node: + image: git.matthiasg.dev/ninluc/selenium/node-uc:latest restart: unless-stopped volumes: - /tmp:/tmp + - chromeProfile:/home/seluser/profile/ shm_size: 2gb + tty: true + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ports: - - "4444:4444" - "7900:7900" + networks: + - selenium-grid + + selenium-hub: + image: selenium/hub:latest + healthcheck: + test: "curl -s http://127.0.0.1:4444/wd/hub/status | jq -e '.value.ready == true' || exit 1" + interval: 30s + timeout: 10s + retries: 3 + restart: unless-stopped + tty: true + ports: + - "4442-4444:4442-4444" + networks: + - default + - selenium-grid watchtower: image: containrrr/watchtower + restart: unless-stopped volumes: - /var/run/docker.sock:/var/run/docker.sock +networks: + selenium-grid: + driver: bridge + volumes: dbdata: + chromeProfile: browserDownloads: browserScreenshots: browserSource: diff --git a/compose.yaml b/compose.yaml index cff1ff8..b00d9c1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,19 +16,19 @@ services: ports: - 80:80 depends_on: - db: - condition: service_healthy - undetected-chromedriver: + # db: + # condition: service_healthy + selenium-hub: condition: service_healthy db: - image: mysql:5.7 + image: mariadb:latest restart: unless-stopped - healthcheck: - test: ["CMD", 'mysqladmin', 'ping', '-h', 'db', '-u', '${DB_USERNAME}', '-p${DB_PASSWORD}' ] - interval: 2s - timeout: 20s - retries: 10 + # healthcheck: + # test: ["CMD", 'mysqladmin', 'ping', '-h', 'db', '-u', '${DB_USERNAME}', '-p${DB_PASSWORD}' ] + # interval: 2s + # timeout: 20s + # retries: 10 environment: MYSQL_ROOT_PASSWORD: ${DB_PASSWORD} MYSQL_DATABASE: ${DB_DATABASE} @@ -37,17 +37,42 @@ services: volumes: - dbdata:/var/lib/mysql - undetected-chromedriver: + undetected-chromedriver-node: build: context: ./undetectedChromedriver dockerfile: seleniumChromedriverDockerfile - restart: unless-stopped volumes: - /tmp:/tmp + - chromeProfile:/home/seluser/profile/ shm_size: 2gb + tty: true + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ports: - - "4444:4444" - "7900:7900" + networks: + - selenium-grid + + selenium-hub: + image: selenium/hub:latest + healthcheck: + test: "curl -s http://localhost:4444/wd/hub/status | jq -e '.value.ready == true' || exit 1" + interval: 30s + timeout: 10s + retries: 3 + tty: true + ports: + - "4442-4444:4442-4444" + networks: + - default + - selenium-grid + +networks: + selenium-grid: + driver: bridge volumes: dbdata: + chromeProfile: diff --git a/undetectedChromedriver/README.md b/undetectedChromedriver/README.md index 288a759..841836f 100644 --- a/undetectedChromedriver/README.md +++ b/undetectedChromedriver/README.md @@ -10,4 +10,15 @@ Could not start a new session. Response code 500. Message: session not created: Go into the `/home/seluser` director in the container and run `rm -rf ~/.config/google-chrome/Singleton*` -If it doesn't work, the log can be found in the set profile folder in the file `chrome_debug.log` +One-liner : `rm -rf /home/seluser/.config/google-chrome/Singleton* && exit` + +If it doesn't work, the log can be found in the seluser chrome profile folder in the file `chrome_debug.log` + +```txt +Could not start a new session. Response code 500. Message: session not created: Chrome instance exited. Examine ChromeDriver verbose log to determine the cause. +``` + +The log can be found in the seluser chrome profile folder in the file `chrome_debug.log`. + +If the log file only says that the chrome profile folder seems to be in use by another chrome application, there is probably a problem with a file in the chrome profile folder like a version change. +The best thing to do is to delete the folder. diff --git a/undetectedChromedriver/chromedriver b/undetectedChromedriver/chromedriver index f65409f..5afbb0a 100755 Binary files a/undetectedChromedriver/chromedriver and b/undetectedChromedriver/chromedriver differ diff --git a/undetectedChromedriver/getChromeDriver.sh b/undetectedChromedriver/getChromeDriver.sh index 63d8808..d94cadc 100644 --- a/undetectedChromedriver/getChromeDriver.sh +++ b/undetectedChromedriver/getChromeDriver.sh @@ -9,17 +9,17 @@ VERSION="latest" # From undetected chromedriver docker #sudo docker run --rm -it -p 3389:3389 -v ./undetectedChromedriver:/root/.local/share/undetected_chromedriver/ ultrafunk/undetected-chromedriver:latest -sudo docker pull selenium/standalone-chrome:$VERSION +sudo docker pull selenium/node-chrome:$VERSION # With undetected chromedriver patcher # Run the selenium/standalone-chrome:latest with a specific container name in the background #sudo docker run -d --name standalone-chrome -v /home/ninluc/Documents/codage/DatBrowser/undetectedChromedriver/chrome/:/opt/google/chrome/ selenium/standalone-chrome:$VERSION -sudo docker run -d --name standalone-chrome selenium/standalone-chrome:$VERSION +sudo docker run -d --name node-chrome selenium/node-chrome:$VERSION sleep 7 # Copy the chromedriver binary from the container to the host -sudo docker cp -L standalone-chrome:/bin/chromedriver ./chromedriver +sudo docker cp -L node-chrome:/bin/chromedriver ./chromedriver sudo chmod 777 ./chromedriver @@ -28,5 +28,5 @@ source venv/bin/activate python3 ./patchChromedriver.py # Stop the container -sudo docker stop standalone-chrome -sudo docker rm standalone-chrome +sudo docker stop node-chrome +sudo docker rm node-chrome diff --git a/undetectedChromedriver/pushSeleniumStandaloneUcImage.sh b/undetectedChromedriver/pushSeleniumStandaloneUcImage.sh index 3fe625f..9b4ab4d 100644 --- a/undetectedChromedriver/pushSeleniumStandaloneUcImage.sh +++ b/undetectedChromedriver/pushSeleniumStandaloneUcImage.sh @@ -1,4 +1,4 @@ #!/bin/bash -sudo docker build -f seleniumChromedriverDockerfile -t git.matthiasg.dev/ninluc/selenium/standalone-uc:latest . -sudo docker push git.matthiasg.dev/ninluc/selenium/standalone-uc:latest +docker build -f seleniumChromedriverDockerfile -t git.matthiasg.dev/ninluc/selenium/node-uc:latest . +docker push git.matthiasg.dev/ninluc/selenium/node-uc:latest diff --git a/undetectedChromedriver/seleniumChromedriverDockerfile b/undetectedChromedriver/seleniumChromedriverDockerfile index 4a68853..7242f97 100644 --- a/undetectedChromedriver/seleniumChromedriverDockerfile +++ b/undetectedChromedriver/seleniumChromedriverDockerfile @@ -1,6 +1,6 @@ # FROM selenium/standalone-chrome:108.0 AS final # FROM selenium/standalone-chrome:133.0-20250606 AS final -FROM selenium/standalone-chrome:latest AS final +FROM selenium/node-chrome:latest AS final COPY ./chromedriver /bin/chromedriver RUN mkdir -p /home/seluser/profile/nigga @@ -9,6 +9,6 @@ ENV TZ=Europe/Brussels # 30 minutes session timeout ENV SE_OPTS="--session-timeout 1800" -HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -s http://localhost:4444/wd/hub/status | jq -e '.value.ready == true' || exit 1 +# HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -s http://localhost:4444/wd/hub/status | jq -e '.value.ready == true' || exit 1 diff --git a/undetectedChromedriver/undetectedChromeDriver.yaml b/undetectedChromedriver/undetectedChromeDriver.yaml index b155f22..4156335 100644 --- a/undetectedChromedriver/undetectedChromeDriver.yaml +++ b/undetectedChromedriver/undetectedChromeDriver.yaml @@ -1,5 +1,22 @@ services: - undetected-chromedriver: + # undetected-chromedriver: + # # image: git.matthiasg.dev/ninluc/selenium/standalone-uc:latest + # image: selenium/standalone-chrome:latest + # # build: + # # context: ./ + # # dockerfile: seleniumChromedriverDockerfile + # volumes: + # - /tmp:/tmp + # - chromeProfile:/home/seluser/profile/ + # shm_size: 2gb + # tty: true + # ports: + # - "4444:4444" + # - "7900:7900" + + undetected-chromedriver-node: + # image: git.matthiasg.dev/ninluc/selenium/standalone-uc:latest + # image: selenium/node-chrome:latest build: context: ./ dockerfile: seleniumChromedriverDockerfile @@ -8,9 +25,27 @@ services: - chromeProfile:/home/seluser/profile/ shm_size: 2gb tty: true + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 ports: - - "4444:4444" - "7900:7900" + networks: + - selenium-grid + + selenium-hub: + image: selenium/hub:latest + tty: true + ports: + - "4442-4444:4442-4444" + networks: + - selenium-grid + + +networks: + selenium-grid: + driver: bridge volumes: chromeProfile: