PostgreSQL Error While Rebuilding

Hello,

When i-rebuild my discourse today via ./launcher rebuild app. i’m receiving this errors:

root@discourse:/opt# cd /var/discourse_docker/
root@discourse:/var/discourse_docker# ./launcher rebuild app
Ensuring launcher is up to date
Fetching origin
Launcher is up-to-date
cd /pups && git pull && /pups/bin/pups --stdin
Already up to date.
I, [2020-07-15T16:31:19.389969 #1]  INFO -- : Loading --stdin
I, [2020-07-15T16:31:19.397738 #1]  INFO -- : > locale-gen $LANG && update-locale
I, [2020-07-15T16:31:19.442371 #1]  INFO -- : Generating locales (this might take a while)...
Generation complete.

I, [2020-07-15T16:31:19.442561 #1]  INFO -- : > mkdir -p /shared/postgres_run
I, [2020-07-15T16:31:19.445577 #1]  INFO -- :
I, [2020-07-15T16:31:19.445781 #1]  INFO -- : > chown postgres:postgres /shared/postgres_run
I, [2020-07-15T16:31:19.449166 #1]  INFO -- :
I, [2020-07-15T16:31:19.449376 #1]  INFO -- : > chmod 775 /shared/postgres_run
I, [2020-07-15T16:31:19.452432 #1]  INFO -- :
I, [2020-07-15T16:31:19.452626 #1]  INFO -- : > rm -fr /var/run/postgresql
I, [2020-07-15T16:31:19.456077 #1]  INFO -- :
I, [2020-07-15T16:31:19.456274 #1]  INFO -- : > ln -s /shared/postgres_run /var/run/postgresql
I, [2020-07-15T16:31:19.459496 #1]  INFO -- :
I, [2020-07-15T16:31:19.459722 #1]  INFO -- : > socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1
2020/07/15 16:31:19 socat[26] E connect(6, AF=1 "/shared/postgres_run/.s.PGSQL.5432", 36): No such file or directory
I, [2020-07-15T16:31:19.466033 #1]  INFO -- :
I, [2020-07-15T16:31:19.466190 #1]  INFO -- : > rm -fr /shared/postgres_run/.s*
I, [2020-07-15T16:31:19.470412 #1]  INFO -- :
I, [2020-07-15T16:31:19.470594 #1]  INFO -- : > rm -fr /shared/postgres_run/*.pid
I, [2020-07-15T16:31:19.474721 #1]  INFO -- :
I, [2020-07-15T16:31:19.474886 #1]  INFO -- : > mkdir -p /shared/postgres_run/12-main.pg_stat_tmp
I, [2020-07-15T16:31:19.477948 #1]  INFO -- :
I, [2020-07-15T16:31:19.478165 #1]  INFO -- : > chown postgres:postgres /shared/postgres_run/12-main.pg_stat_tmp
I, [2020-07-15T16:31:19.481360 #1]  INFO -- :
I, [2020-07-15T16:31:19.488377 #1]  INFO -- : File > /etc/service/postgres/run  chmod: +x  chown:
I, [2020-07-15T16:31:19.494864 #1]  INFO -- : File > /etc/service/postgres/log/run  chmod: +x  chown:
I, [2020-07-15T16:31:19.501433 #1]  INFO -- : File > /etc/runit/3.d/99-postgres  chmod: +x  chown:
I, [2020-07-15T16:31:19.508030 #1]  INFO -- : File > /root/upgrade_postgres  chmod: +x  chown:
I, [2020-07-15T16:31:19.508311 #1]  INFO -- : > chown -R root /var/lib/postgresql/12/main
I, [2020-07-15T16:31:22.627505 #1]  INFO -- :
I, [2020-07-15T16:31:22.627827 #1]  INFO -- : > [ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/12/bin/initdb -D /shared/postgres_data || exit 0
I, [2020-07-15T16:31:22.631480 #1]  INFO -- :
I, [2020-07-15T16:31:22.631656 #1]  INFO -- : > chown -R postgres:postgres /shared/postgres_data
I, [2020-07-15T16:31:22.649989 #1]  INFO -- :
I, [2020-07-15T16:31:22.650148 #1]  INFO -- : > chown -R postgres:postgres /var/run/postgresql
I, [2020-07-15T16:31:22.653807 #1]  INFO -- :
I, [2020-07-15T16:31:22.654052 #1]  INFO -- : > /root/upgrade_postgres
I, [2020-07-15T16:31:22.660478 #1]  INFO -- :
I, [2020-07-15T16:31:22.660651 #1]  INFO -- : > rm /root/upgrade_postgres
I, [2020-07-15T16:31:22.663338 #1]  INFO -- :
I, [2020-07-15T16:31:22.663767 #1]  INFO -- : Replacing data_directory = '/var/lib/postgresql/12/main' with data_directory = '/shared/postgres_data' in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.664582 #1]  INFO -- : Replacing (?-mix:#?listen_addresses *=.*) with listen_addresses = '*' in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.666768 #1]  INFO -- : Replacing (?-mix:#?synchronous_commit *=.*) with synchronous_commit = $db_synchronous_commit in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.668210 #1]  INFO -- : Replacing (?-mix:#?shared_buffers *=.*) with shared_buffers = $db_shared_buffers in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.669770 #1]  INFO -- : Replacing (?-mix:#?work_mem *=.*) with work_mem = $db_work_mem in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.671341 #1]  INFO -- : Replacing (?-mix:#?default_text_search_config *=.*) with default_text_search_config = '$db_default_text_search_config' in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.673141 #1]  INFO -- : > install -d -m 0755 -o postgres -g postgres /shared/postgres_backup
I, [2020-07-15T16:31:22.676724 #1]  INFO -- :
I, [2020-07-15T16:31:22.677097 #1]  INFO -- : Replacing (?-mix:#?max_wal_senders *=.*) with max_wal_senders = $db_max_wal_senders in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.677542 #1]  INFO -- : Replacing (?-mix:#?wal_level *=.*) with wal_level = $db_wal_level in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.679169 #1]  INFO -- : Replacing (?-mix:#?checkpoint_segments *=.*) with checkpoint_segments = $db_checkpoint_segments in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.680967 #1]  INFO -- : Replacing (?-mix:#?logging_collector *=.*) with logging_collector = $db_logging_collector in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.682498 #1]  INFO -- : Replacing (?-mix:#?log_min_duration_statement *=.*) with log_min_duration_statement = $db_log_min_duration_statement in /etc/postgresql/12/main/postgresql.conf
I, [2020-07-15T16:31:22.684043 #1]  INFO -- : Replacing (?-mix:^#local +replication +postgres +peer$) with local replication postgres  peer in /etc/postgresql/12/main/pg_hba.conf
I, [2020-07-15T16:31:22.684445 #1]  INFO -- : Replacing (?-mix:^host.*all.*all.*127.*$) with host all all 0.0.0.0/0 md5 in /etc/postgresql/12/main/pg_hba.conf
I, [2020-07-15T16:31:22.685537 #1]  INFO -- : > HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/12/bin/postmaster -D /etc/postgresql/12/main
I, [2020-07-15T16:31:22.687451 #1]  INFO -- : > sleep 5
2020-07-15 16:31:22.703 GMT [49] LOG:  syntax error in file "/etc/postgresql/12/main/postgresql.conf" line 193, near token "$"
2020-07-15 16:31:22.703 GMT [49] LOG:  syntax error in file "/etc/postgresql/12/main/postgresql.conf" line 286, near token "$"
2020-07-15 16:31:22.703 GMT [49] FATAL:  configuration file "/etc/postgresql/12/main/postgresql.conf" contains errors
I, [2020-07-15T16:31:27.690822 #1]  INFO -- :
I, [2020-07-15T16:31:27.691358 #1]  INFO -- : > su postgres -c 'createdb discourse' || true
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T16:31:27.755611 #1]  INFO -- :
I, [2020-07-15T16:31:27.755836 #1]  INFO -- : > su postgres -c 'psql discourse -c "create user discourse;"' || true
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T16:31:27.811853 #1]  INFO -- :
I, [2020-07-15T16:31:27.812171 #1]  INFO -- : > su postgres -c 'psql discourse -c "grant all privileges on database discourse to discourse;"' || true
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T16:31:27.865404 #1]  INFO -- :
I, [2020-07-15T16:31:27.865605 #1]  INFO -- : > su postgres -c 'psql discourse -c "alter schema public owner to discourse;"'
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T16:31:27.917261 #1]  INFO -- :
I, [2020-07-15T16:31:27.917634 #1]  INFO -- : Terminating async processes


FAILED
--------------------
Pups::ExecError: su postgres -c 'psql discourse -c "alter schema public owner to discourse;"' failed with return #<Process::Status: pid 61 exit 1>
Location of failure: /pups/lib/pups/exec_command.rb:112:in `spawn'
exec failed with the params "su postgres -c 'psql $db_name -c \"alter schema public owner to $db_user;\"'"
7670f9fb0161c9f213e14e09345518109b1ec04131d0428375f112ffccb98337
** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one.
./discourse-doctor may help diagnose the problem.

Also tried to just git clone again GitHub - discourse/discourse_docker: A Docker image for Discourse and error still persist? any help? discourse_doctor doesn’t help.

Tried to fresh install discourse and error still persist.

Is this a PostgreSQL 12 upgrade error @Falco?

Same issue for a fresh install on EC2 (Ubuntu 18.04.4 LTS) following the cloud installation instructions just now:

    I, [2020-07-15T16:37:49.240541 #1]  INFO -- : > mkdir -p /shared/postgres_run
    I, [2020-07-15T16:37:49.242993 #1]  INFO -- :
    I, [2020-07-15T16:37:49.243332 #1]  INFO -- : > chown postgres:postgres /shared/postgres_run
    I, [2020-07-15T16:37:49.245340 #1]  INFO -- :
    I, [2020-07-15T16:37:49.245758 #1]  INFO -- : > chmod 775 /shared/postgres_run
    I, [2020-07-15T16:37:49.247483 #1]  INFO -- :
    I, [2020-07-15T16:37:49.247744 #1]  INFO -- : > rm -fr /var/run/postgresql
    I, [2020-07-15T16:37:49.249718 #1]  INFO -- :
    I, [2020-07-15T16:37:49.250109 #1]  INFO -- : > ln -s /shared/postgres_run /var/run/postgresql
    I, [2020-07-15T16:37:49.251780 #1]  INFO -- :
    I, [2020-07-15T16:37:49.252044 #1]  INFO -- : > socat /dev/null UNIX-CONNECT:/shared/postgres_run/.s.PGSQL.5432 || exit 0 && echo postgres already running stop container ; exit 1
    2020/07/15 16:37:49 socat[26] E connect(6, AF=1 "/shared/postgres_run/.s.PGSQL.5432", 36): No such file or directory
    I, [2020-07-15T16:37:49.255669 #1]  INFO -- :
    I, [2020-07-15T16:37:49.255932 #1]  INFO -- : > rm -fr /shared/postgres_run/.s*
    I, [2020-07-15T16:37:49.258317 #1]  INFO -- :
    I, [2020-07-15T16:37:49.258498 #1]  INFO -- : > rm -fr /shared/postgres_run/*.pid
    I, [2020-07-15T16:37:49.260852 #1]  INFO -- :
    I, [2020-07-15T16:37:49.261155 #1]  INFO -- : > mkdir -p /shared/postgres_run/12-main.pg_stat_tmp
    I, [2020-07-15T16:37:49.263194 #1]  INFO -- :
    I, [2020-07-15T16:37:49.263496 #1]  INFO -- : > chown postgres:postgres /shared/postgres_run/12-main.pg_stat_tmp
    I, [2020-07-15T16:37:49.265228 #1]  INFO -- :
    I, [2020-07-15T16:37:49.269169 #1]  INFO -- : File > /etc/service/postgres/run  chmod: +x  chown:
    I, [2020-07-15T16:37:49.272879 #1]  INFO -- : File > /etc/service/postgres/log/run  chmod: +x  chown:
    I, [2020-07-15T16:37:49.276415 #1]  INFO -- : File > /etc/runit/3.d/99-postgres  chmod: +x  chown:
    I, [2020-07-15T16:37:49.280134 #1]  INFO -- : File > /root/upgrade_postgres  chmod: +x  chown:
    I, [2020-07-15T16:37:49.280510 #1]  INFO -- : > chown -R root /var/lib/postgresql/12/main
    I, [2020-07-15T16:37:51.389258 #1]  INFO -- :
    I, [2020-07-15T16:37:51.389481 #1]  INFO -- : > [ ! -e /shared/postgres_data ] && install -d -m 0755 -o postgres -g postgres /shared/postgres_data && sudo -E -u postgres /usr/lib/postgresql/12/bin/initdb -D /shared/postgres_data || exit 0
    I, [2020-07-15T16:37:51.393651 #1]  INFO -- :
    I, [2020-07-15T16:37:51.393783 #1]  INFO -- : > chown -R postgres:postgres /shared/postgres_data
    I, [2020-07-15T16:37:51.401336 #1]  INFO -- :
    I, [2020-07-15T16:37:51.405711 #1]  INFO -- : > chown -R postgres:postgres /var/run/postgresql
    I, [2020-07-15T16:37:51.409068 #1]  INFO -- :
    I, [2020-07-15T16:37:51.409759 #1]  INFO -- : > /root/upgrade_postgres
    I, [2020-07-15T16:37:51.417131 #1]  INFO -- :
    I, [2020-07-15T16:37:51.417936 #1]  INFO -- : > rm /root/upgrade_postgres
    I, [2020-07-15T16:37:51.420632 #1]  INFO -- :
    I, [2020-07-15T16:37:51.421270 #1]  INFO -- : Replacing data_directory = '/var/lib/postgresql/12/main' with data_directory = '/shared/postgres_data' in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.421970 #1]  INFO -- : Replacing (?-mix:#?listen_addresses *=.*) with listen_addresses = '*' in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.423148 #1]  INFO -- : Replacing (?-mix:#?synchronous_commit *=.*) with synchronous_commit = $db_synchronous_commit in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.424102 #1]  INFO -- : Replacing (?-mix:#?shared_buffers *=.*) with shared_buffers = $db_shared_buffers in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.425086 #1]  INFO -- : Replacing (?-mix:#?work_mem *=.*) with work_mem = $db_work_mem in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.426032 #1]  INFO -- : Replacing (?-mix:#?default_text_search_config *=.*) with default_text_search_config = '$db_default_text_search_config' in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.426983 #1]  INFO -- : > install -d -m 0755 -o postgres -g postgres /shared/postgres_backup
    I, [2020-07-15T16:37:51.430084 #1]  INFO -- :
    I, [2020-07-15T16:37:51.430573 #1]  INFO -- : Replacing (?-mix:#?max_wal_senders *=.*) with max_wal_senders = $db_max_wal_senders in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.431292 #1]  INFO -- : Replacing (?-mix:#?wal_level *=.*) with wal_level = $db_wal_level in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.433312 #1]  INFO -- : Replacing (?-mix:#?checkpoint_segments *=.*) with checkpoint_segments = $db_checkpoint_segments in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.434127 #1]  INFO -- : Replacing (?-mix:#?logging_collector *=.*) with logging_collector = $db_logging_collector in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.435128 #1]  INFO -- : Replacing (?-mix:#?log_min_duration_statement *=.*) with log_min_duration_statement = $db_log_min_duration_statement in /etc/postgresql/12/main/postgresql.conf
    I, [2020-07-15T16:37:51.436084 #1]  INFO -- : Replacing (?-mix:^#local +replication +postgres +peer$) with local replication postgres  peer in /etc/postgresql/12/main/pg_hba.conf
    I, [2020-07-15T16:37:51.436566 #1]  INFO -- : Replacing (?-mix:^host.*all.*all.*127.*$) with host all all 0.0.0.0/0 md5 in /etc/postgresql/12/main/pg_hba.conf
    I, [2020-07-15T16:37:51.437521 #1]  INFO -- : > HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/12/bin/postmaster -D /etc/postgresql/12/main
    I, [2020-07-15T16:37:51.440165 #1]  INFO -- : > sleep 5
    2020-07-15 16:37:51.457 GMT [49] LOG:  syntax error in file "/etc/postgresql/12/main/postgresql.conf" line 193, near token "$"
    2020-07-15 16:37:51.457 GMT [49] LOG:  syntax error in file "/etc/postgresql/12/main/postgresql.conf" line 286, near token "$"
    2020-07-15 16:37:51.457 GMT [49] FATAL:  configuration file "/etc/postgresql/12/main/postgresql.conf" contains errors
    I, [2020-07-15T16:37:56.451144 #1]  INFO -- :
    I, [2020-07-15T16:37:56.451405 #1]  INFO -- : > su postgres -c 'createdb discourse' || true
    Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
    I, [2020-07-15T16:37:56.503071 #1]  INFO -- :
    I, [2020-07-15T16:37:56.503968 #1]  INFO -- : > su postgres -c 'psql discourse -c "create user discourse;"' || true
    Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
    I, [2020-07-15T16:37:56.550717 #1]  INFO -- :
    I, [2020-07-15T16:37:56.550930 #1]  INFO -- : > su postgres -c 'psql discourse -c "grant all privileges on database discourse to discourse;"' || true
    Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
    I, [2020-07-15T16:37:56.612944 #1]  INFO -- :
    I, [2020-07-15T16:37:56.613161 #1]  INFO -- : > su postgres -c 'psql discourse -c "alter schema public owner to discourse;"'
    Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
    I, [2020-07-15T16:37:56.665397 #1]  INFO -- :
    I, [2020-07-15T16:37:56.665800 #1]  INFO -- : Terminating async processes


    FAILED
    --------------------
    Pups::ExecError: su postgres -c 'psql discourse -c "alter schema public owner to discourse;"' failed with return #<Process::Status: pid 61 exit 1>
    Location of failure: /pups/lib/pups/exec_command.rb:112:in `spawn'
    exec failed with the params "su postgres -c 'psql $db_name -c \"alter schema public owner to $db_user;\"'"

Exactly the same problem on fresh DO server

I am receiving this when rebuilding.

I was able to rebuild by adding the following params to templates/postgres.template.yml:

db_wal_level: minimal
db_max_wal_senders: 0

Yes. the new commit on master removed this line on all postgresql

A smoke test will be enough to check this PR if it broke master branch.

Fix confirmed for fresh install. Thank you!

templates/postgres.template.yml:

params:
  db_synchronous_commit: "off"
  db_shared_buffers: "256MB"
  db_work_mem: "10MB"
  db_default_text_search_config: "pg_catalog.english"
  db_name: discourse
  db_user: discourse
  db_checkpoint_segments: 6
  db_logging_collector: off
  db_log_min_duration_statement: 100
  db_wal_level: minimal
  db_max_wal_senders: 0

It looks like this didn’t do it. I’ve got that commit and I still get:

Oh. I might be confused.

I, [2020-07-15T17:11:53.608333 #1]  INFO -- : > HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/12/bin/postmaster -D /etc/postgresql/12/main
I, [2020-07-15T17:11:53.610211 #1]  INFO -- : > sleep 5
2020-07-15 17:11:53.641 GMT [49] LOG:  syntax error in file "/etc/postgresql/12/main/postgresql.conf" line 193, near token "$" 
2020-07-15 17:11:53.642 GMT [49] LOG:  syntax error in file "/etc/postgresql/12/main/postgresql.conf" line 286, near token "$"
2020-07-15 17:11:53.643 GMT [49] FATAL:  configuration file "/etc/postgresql/12/main/postgresql.conf" contains errors
I, [2020-07-15T17:11:58.619578 #1]  INFO -- :
I, [2020-07-15T17:11:58.619911 #1]  INFO -- : > su postgres -c 'createdb discourse' || true
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T17:11:58.700776 #1]  INFO -- :
I, [2020-07-15T17:11:58.701730 #1]  INFO -- : > su postgres -c 'psql discourse -c "create user discourse;"' || true
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T17:11:58.775094 #1]  INFO -- :
I, [2020-07-15T17:11:58.776049 #1]  INFO -- : > su postgres -c 'psql discourse -c "grant all privileges on database discourse to discourse;"' || true
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T17:11:58.847143 #1]  INFO -- :
I, [2020-07-15T17:11:58.848265 #1]  INFO -- : > su postgres -c 'psql discourse -c "alter schema public owner to discourse;"'
Error: invalid line 193 in /etc/postgresql/12/main/postgresql.conf: wal_level = $db_wal_level
I, [2020-07-15T17:11:58.918991 #1]  INFO -- :
I, [2020-07-15T17:11:58.920088 #1]  INFO -- : Terminating async processes

But it looks like that got merged and

needs to have $db_wal_level defined.

Sorry about that! That was meant to be removed entirely. Fixed.

The new commit fixed it. thanks mate. also test your thing first before merging it to master branch :+1: