30b41029c1
* setup redis infra for appview indexer * barebones bsky ingester * add ioredis to bsky * remove some indexer functionality from bsky api * setup for bsky indexer * tidy * tidy, observe basic pipeline functioning * process messages on bsky indexer pipeline, tidy tests and lifecycle * trim partitions when moving cursor * simplify config for partitions * misc fixes for redis setup in bsky tests, add namespacing * fix pds proxy tests * remove cursor state from indexer partitions, simplify ingester state * tidy * utils for testing w/ multiple indexers, fix off-by-one xtrim * test reingesting * test indexer repartitioning * add entrypoints for bsky ingester and indexer, fix db schema config, api entrypoint name, tidy * setup and test bsky ingester backpressure, add config * tidy * add missing test file * tidy redis calls, add redis sentinel config * tidy/test some utils used in bsky pipeline * tidy bsky pipeline tests, move helpers into dev-env * fix pds crud test * support redis host and password config * better loggin/observability in ingester and indexer, make build * add bsky ingester initial cursor config * temporarily remove migrations from indexer/ingester * allow ingester to batch * packages/pg becomes packages/dev-infra with some cleanup (#1402) * packages/dev-infra/ * Extract packages/dev-infra/_common.sh and use it * Use period instead of source because of /bin/sh * add docs for redis test script * fix repartition test * add logs to debug ci * simplify repartitioning test, remove ci logs --------- Co-authored-by: Jerry Chen <jerry@redelm.net>
50 lines
1.2 KiB
YAML
50 lines
1.2 KiB
YAML
version: '3.8'
|
|
services:
|
|
# An ephermerally-stored postgres database for single-use test runs
|
|
db_test: &db_test
|
|
image: postgres:14.4-alpine
|
|
environment:
|
|
- POSTGRES_USER=pg
|
|
- POSTGRES_PASSWORD=password
|
|
ports:
|
|
- '5433:5432'
|
|
# Healthcheck ensures db is queryable when `docker-compose up --wait` completes
|
|
healthcheck:
|
|
test: 'pg_isready -U pg'
|
|
interval: 500ms
|
|
timeout: 10s
|
|
retries: 20
|
|
# A persistently-stored postgres database
|
|
db:
|
|
<<: *db_test
|
|
ports:
|
|
- '5432:5432'
|
|
healthcheck:
|
|
disable: true
|
|
volumes:
|
|
- atp_db:/var/lib/postgresql/data
|
|
# An ephermerally-stored redis cache for single-use test runs
|
|
redis_test: &redis_test
|
|
image: redis:7.0-alpine
|
|
ports:
|
|
- '6380:6379'
|
|
# Healthcheck ensures redis is queryable when `docker-compose up --wait` completes
|
|
healthcheck:
|
|
test: ['CMD-SHELL', '[ "$$(redis-cli ping)" = "PONG" ]']
|
|
interval: 500ms
|
|
timeout: 10s
|
|
retries: 20
|
|
# A persistently-stored redis cache
|
|
redis:
|
|
<<: *redis_test
|
|
command: redis-server --save 60 1 --loglevel warning
|
|
ports:
|
|
- '6379:6379'
|
|
healthcheck:
|
|
disable: true
|
|
volumes:
|
|
- atp_redis:/data
|
|
volumes:
|
|
atp_db:
|
|
atp_redis:
|