diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml new file mode 100644 index 0000000..922b8f0 --- /dev/null +++ b/.github/workflows/backend-ci.yml @@ -0,0 +1,91 @@ +name: Backend CI + +on: + push: + branches: [main, master] + paths: + - 'backend/**' + - '.github/workflows/backend-ci.yml' + pull_request: + branches: [main, master] + paths: + - 'backend/**' + - '.github/workflows/backend-ci.yml' + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + check: + runs-on: ubuntu-latest + defaults: + run: + working-directory: backend + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-action@stable + with: + components: rustfmt, clippy + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + backend/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Check formatting + run: cargo fmt --all -- --check + + - name: Clippy + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Build + run: cargo build --release + + - name: Run tests + run: cargo test --release + + build-binaries: + runs-on: ubuntu-latest + needs: check + defaults: + run: + working-directory: backend + + steps: + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-action@stable + + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + backend/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build release binaries + run: cargo build --release + + - name: Upload binaries + uses: actions/upload-artifact@v4 + with: + name: backend-binaries + path: | + backend/target/release/telemetry_core + backend/target/release/telemetry_shard + retention-days: 7 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..13860b9 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,98 @@ +name: Docker Build + +on: + push: + branches: [main, master] + tags: + - 'v*' + pull_request: + branches: [main, master] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + BACKEND_IMAGE: ghcr.io/${{ github.repository }}-backend + FRONTEND_IMAGE: ghcr.io/${{ github.repository }}-frontend + +jobs: + build-backend: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.BACKEND_IMAGE }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=sha + + - name: Build and push backend + uses: docker/build-push-action@v5 + with: + context: ./backend + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-frontend: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.FRONTEND_IMAGE }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=sha + + - name: Build and push frontend + uses: docker/build-push-action@v5 + with: + context: ./frontend + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/frontend-ci.yml b/.github/workflows/frontend-ci.yml new file mode 100644 index 0000000..6976b59 --- /dev/null +++ b/.github/workflows/frontend-ci.yml @@ -0,0 +1,53 @@ +name: Frontend CI + +on: + push: + branches: [main, master] + paths: + - 'frontend/**' + - '.github/workflows/frontend-ci.yml' + pull_request: + branches: [main, master] + paths: + - 'frontend/**' + - '.github/workflows/frontend-ci.yml' + workflow_dispatch: + +jobs: + lint-and-test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Lint + run: npm run lint + + - name: Prettier check + run: npm run pretty:check + + - name: Run tests + run: npm run test -- --passWithNoTests + + - name: Build + run: npm run build + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: frontend-build + path: frontend/build/ + retention-days: 7 diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 0000000..8df7b46 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,23 @@ +name: Lock Threads + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v5 + with: + github-token: ${{ github.token }} + issue-inactive-days: 30 + pr-inactive-days: 30 + issue-lock-reason: resolved + pr-lock-reason: resolved + log-output: true diff --git a/README.md b/README.md index 5f895b4..a2823fe 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,19 @@ -![Frontend](https://github.com/paritytech/substrate-telemetry/workflows/Frontend%20CI/badge.svg) -![Backend](https://github.com/paritytech/substrate-telemetry/workflows/Backend%20CI/badge.svg) - -# Polkadot Telemetry +# Pezkuwichain Telemetry ## Overview -This repository contains the backend ingestion server for Substrate Telemetry (which itself is comprised of two binaries; `telemetry_shard` and `telemetry_core`) as well as the Frontend you typically see running at [telemetry.polkadot.io](https://telemetry.polkadot.io/). +This repository contains the Telemetry server for Pezkuwichain nodes. It includes: -The backend is a Rust project and the frontend is React/Typescript project. +- **Backend**: Rust-based ingestion server (`telemetry_shard` and `telemetry_core`) +- **Frontend**: React/TypeScript dashboard for monitoring network nodes -Substrate based nodes can be connected to an arbitrary Telemetry backend using the `--telemetry-url` (see below for more complete instructions on how to get this up and running). +Pezkuwichain nodes can connect to Telemetry using the `--telemetry-url` flag. ### Messages -Depending on the configured verbosity, substrate nodes will send different types of messages to the Telemetry server. Verbosity level `0` is sufficient to provide the Telemetry server with almost all of the node information needed. Using this verbosity level will lead to the substrate node sending the following message types to Telemetry: +Nodes send telemetry messages depending on the verbosity level: +**Verbosity 0 (default):** ``` system.connected system.interval @@ -22,202 +21,118 @@ block.import notify.finalized ``` -Increasing the verbosity level to 1 will lead to additional "consensus info" messages being sent, one of which has the identifier: - +**Verbosity 1 (includes validator address):** ``` afg.authority_set ``` -Which we use to populate the "validator address" field if applicable. - -Increasing the verbosity level beyond 1 is unnecessary, and will not result in any additional messages that Telemetry can handle (but other metric gathering systems might find them useful). - ## Getting Started -To run the backend, you will need `cargo` to build the binary. We recommend using [`rustup`](https://rustup.rs/). +### Prerequisites -To run the frontend make sure to grab the latest stable version of node and install dependencies before doing anything: +- Rust (via rustup): https://rustup.rs/ +- Node.js (LTS version) -```sh -nvm install stable -(cd frontend && npm install) -``` +### Backend -### Terminal 1 & 2 - Backend - -Build the backend binaries by running the following: - -``` +```bash cd backend cargo build --release -``` -And then, in two different terminals, run: - -``` +# Terminal 1 - Core server ./target/release/telemetry_core -``` -and - -``` +# Terminal 2 - Shard server ./target/release/telemetry_shard ``` -Use `--help` on either binary to see the available options. +Default ports: +- `telemetry_core`: 127.0.0.1:8000 +- `telemetry_shard`: 127.0.0.1:8001 -By default, `telemetry_core` will listen on 127.0.0.1:8000, and `telemetry_shard` will listen on 127.0.0.1:8001, and expect the `telemetry_core` to be listening on its default address. To listen on different addresses, use the `--listen` option on either binary, for example `--listen 0.0.0.0:8000`. The `telemetry_shard` also needs to be told where the core is, so if the core is configured with `--listen 127.0.0.1:9090`, remember to pass `--core 127.0.0.1:9090` to the shard, too. +### Frontend -### Terminal 3 - Frontend - -```sh +```bash cd frontend npm install npm run start ``` -Once this is running, you'll be able to navigate to [http://localhost:3000](http://localhost:3000) to view the UI. +Access the UI at http://localhost:3000 -### Terminal 4 - Node +### Connect a Node -Follow up installation instructions from the [Polkadot repo](https://github.com/paritytech/polkadot) - -If you started the backend binaries with their default arguments, you can connect a node to the shard by running: - -```sh -polkadot --dev --telemetry-url 'ws://localhost:8001/submit 0' +```bash +pezkuwi --dev --telemetry-url 'ws://localhost:8001/submit 0' ``` -**Note:** The "0" at the end of the URL is a verbosity level, and not part of the URL itself. Verbosity levels range from 0-9, with 0 denoting the lowest verbosity. The URL and this verbosity level are parts of a single argument and must therefore be surrounded in quotes (as seen above) in order to be treated as such by your shell. +Note: The "0" is the verbosity level (0-9). ## Docker -### Building images +### Build Images -To build the backend docker image, navigate into the `backend` folder of this repository and run: +```bash +# Backend +cd backend +docker build -t pezkuwichain/telemetry-backend . -``` -docker build -t parity/substrate-telemetry-backend . +# Frontend +cd frontend +docker build -t pezkuwichain/telemetry-frontend . ``` -The backend image contains both the `telemetry_core` and `telemetry_shard` binaries. +### Docker Compose -To build the frontend docker image, navigate into the `frontend` folder and run: - -``` -docker build -t parity/substrate-telemetry-frontend . +```bash +docker-compose up ``` -### Run the backend and frontend using `docker-compose` +### Manual Docker Setup -The easiest way to run the backend and frontend images is to use `docker-compose`. To do this, run `docker-compose up` in the root of this repository to build and run the images. Once running, you can view the UI by navigating a browser to `http://localhost:3000`. +```bash +# Create network +docker network create telemetry -To connect a substrate node and have it send telemetry to this running instance, you have to tell it where to send telemetry by appending the argument `--telemetry-url 'ws://localhost:8001/submit 0'` (see "Terminal 4 - Node" above). +# Backend core +docker run -d --network=telemetry --name backend-core \ + -p 8000:8000 --read-only \ + pezkuwichain/telemetry-backend \ + telemetry_core -l 0.0.0.0:8000 -### Run the backend and frontend using `docker` +# Backend shard +docker run -d --network=telemetry --name backend-shard \ + -p 8001:8001 --read-only \ + pezkuwichain/telemetry-backend \ + telemetry_shard -l 0.0.0.0:8001 -c http://backend-core:8000/shard_submit -If you'd like to get things running manually using Docker, you can do the following. This assumes that you've built the images as per the above, and have two images named `parity/substrate-telemetry-backend` and `parity/substrate-telemetry-frontend`. - -1. Create a new shared network so that the various containers can communicate with eachother: - - ``` - docker network create telemetry - ``` - -2. Start up the backend core process. We expose port 8000 so that a UI running in a host browser can connect to the `/feed` endpoint. - - ``` - docker run --rm -it --network=telemetry \ - --name backend-core \ - -p 8000:8000 \ - --read-only \ - parity/substrate-telemetry-backend \ - telemetry_core -l 0.0.0.0:8000 - ``` - -3. In another terminal, start up the backend shard process. We tell it where it can reach the core to send messages (possible because it has been started on the same network), and we listen on and expose port 8001 so that nodes running in the host can connect and send telemetry to it. - - ``` - docker run --rm -it --network=telemetry \ - --name backend-shard \ - -p 8001:8001 \ - --read-only \ - parity/substrate-telemetry-backend \ - telemetry_shard -l 0.0.0.0:8001 -c http://backend-core:8000/shard_submit - ``` - -4. In another terminal, start up the frontend server. We pass a `SUBSTRATE_TELEMETRY_URL` env var to tell the UI how to connect to the core process to receive telemetry. This is relative to the host machine, since that is where the browser and UI will be running. - - ``` - docker run --rm -it --network=telemetry \ - --name frontend \ - -p 3000:8000 \ - -e SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed \ - parity/substrate-telemetry-frontend - ``` - - **NOTE:** Here we used `SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed`. This will work if you test with everything running locally on your machine but NOT if your backend runs on a remote server. Keep in mind that the frontend docker image is serving a static site running your browser. The `SUBSTRATE_TELEMETRY_URL` is the WebSocket url that your browser will use to reach the backend. Say your backend runs on a remote server at `foo.example.com`, you will need to set the IP/url accordingly in `SUBSTRATE_TELEMETRY_URL` (in this case, to `ws://foo.example.com/feed`). - - **NOTE:** Running the frontend container in *read-only* mode reduces attack surface that could be used to exploit - a container. It requires however a little more effort and mounting additional volumes as shown below: - - ``` - docker run --rm -it -p 80:8000 --name frontend \ - -e SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed \ - --tmpfs /var/cache/nginx:uid=101,gid=101 \ - --tmpfs /var/run:uid=101,gid=101 \ - --tmpfs /app/tmp:uid=101,gid=101 \ - --read-only \ - parity/substrate-telemetry-frontend - ``` - -With these running, you'll be able to navigate to [http://localhost:3000](http://localhost:3000) to view the UI. If you'd like to connect a node and have it send telemetry to your running shard, you can run the following: - -```sh -docker run --rm -it --network=telemetry \ - --name substrate \ - -p 9944:9944 \ - chevdor/substrate \ - substrate --dev --telemetry-url 'ws://backend-shard:8001/submit 0' +# Frontend +docker run -d --network=telemetry --name frontend \ + -p 3000:8000 \ + -e SUBSTRATE_TELEMETRY_URL=ws://localhost:8000/feed \ + pezkuwichain/telemetry-frontend ``` -You should now see your node showing up in your local [telemetry frontend](http://localhost:3000/): +## Configuration -![image](doc/screenshot01.png) +### Environment Variables -## Deployment +| Variable | Description | Default | +|----------|-------------|---------| +| `SUBSTRATE_TELEMETRY_URL` | WebSocket URL for frontend to connect | `ws://localhost:8000/feed` | -This section covers the internal deployment of Substrate Telemetry to our staging and live environments. +## Links -### Deployment to staging +- **Website**: https://pezkuwichain.io +- **Telemetry**: https://telemetry.pezkuwichain.io +- **Discord**: https://discord.gg/Y3VyEC6h8W -Every time new code is merged to `master`, a new version of telemetry will be automatically built and deployed to our staging environment, so there is nothing that you need to do. Roughly what will happen is: +## License -- An image tag will be generated that looks like `master-$CI_COMMIT_SHORT_SHA`, for example `master-224b1fae`. -- Docker images for the frontend and backend will be pushed to the docker repo (see https://hub.docker.com/r/parity/substrate-telemetry-backend/tags?page=1&ordering=last_updated and https://hub.docker.com/r/parity/substrate-telemetry-frontend/tags?page=1&ordering=last_updated). -- A deployment to the staging environment will be performed using these images. Go to https://github.com/paritytech/substrate-telemetry/actions to inspect the progress of such deployments. +GPL-3.0 -### Deployment to live +## Credits -Once we're happy with things in staging, we can do a deployment to live as follows: +Based on [substrate-telemetry](https://github.com/paritytech/substrate-telemetry) by Parity Technologies. -1. Ensure that the PRs you'd like to deploy are merged to master. -2. Tag the commit on `master` that you'd like to deploy with the form `v1.0-a1b2c3d`. - - The version number (`1.0` here) should just be incremented from whatever the latest version found using `git tag` is. We don't use semantic versioning or anything like that; this is just a dumb "increment version number" approach so that we can see clearly what we've deployed to live and in what order. - - The suffix is a short git commit hash (which can be generated with `git rev-parse --short HEAD`), just so that it's really easy to relate the built docker images back to the corresponding code. -3. Pushing the tag (eg `git push origin v1.0-a1b2c3d`) will kick off the deployment process, which in this case will also lead to new docker images being built. You can view the progress at https://github.com/paritytech/substrate-telemetry/actions. - -> [!WARNING] -> After the tag is pushed the deploy will be done to both environments (staging and production) automatically. - -4. Once a deployment to staging has been successful, run whatever tests you need against the staging deployment to convince yourself that you're happy with it. -5. Confirm that things are working once the deployment has finished by visiting https://telemetry.polkadot.io/. - -### Rolling back to a previous deployment - -If something goes wrong running the above, we can roll back the deployment to live as follows. - -1. Decide what image tag you'd like to roll back to. Push the tag to the commit you want to revert to. -2. Navigate to https://github.com/paritytech/substrate-telemetry/actions and check that the pipeline is successful. -3. Confirm that things are working once the deployment has finished by visiting https://telemetry.polkadot.io/. -4. In case of issues contact devops team. +Maintained by Dijital Kurdistan Tech Institute. diff --git a/backend/common/Cargo.toml b/backend/common/Cargo.toml index 2affa37..60c1e60 100644 --- a/backend/common/Cargo.toml +++ b/backend/common/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "common" version = "0.1.0" -authors = ["Parity Technologies Ltd. "] +authors = ["Dijital Kurdistan Tech Institute "] edition = "2021" license = "GPL-3.0" diff --git a/backend/telemetry_core/Cargo.toml b/backend/telemetry_core/Cargo.toml index 79485dc..466e3d5 100644 --- a/backend/telemetry_core/Cargo.toml +++ b/backend/telemetry_core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "telemetry_core" version = "0.1.0" -authors = ["Parity Technologies Ltd. "] +authors = ["Dijital Kurdistan Tech Institute "] edition = "2021" license = "GPL-3.0" diff --git a/backend/telemetry_shard/Cargo.toml b/backend/telemetry_shard/Cargo.toml index edc52bd..65592d6 100644 --- a/backend/telemetry_shard/Cargo.toml +++ b/backend/telemetry_shard/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "telemetry_shard" version = "0.1.0" -authors = ["Parity Technologies Ltd. "] +authors = ["Dijital Kurdistan Tech Institute "] edition = "2021" license = "GPL-3.0" diff --git a/frontend/assets/favicon.svg b/frontend/assets/favicon.svg index b16d3e3..b387bb8 100644 --- a/frontend/assets/favicon.svg +++ b/frontend/assets/favicon.svg @@ -1,5 +1,4 @@ - - - - + + + P diff --git a/frontend/assets/index.html b/frontend/assets/index.html index be1c24a..45490cb 100644 --- a/frontend/assets/index.html +++ b/frontend/assets/index.html @@ -7,7 +7,7 @@ content="width=device-width, initial-scale=1, shrink-to-fit=no" /> - Polkadot Telemetry + Pezkuwichain Telemetry