scripts: don't export the default attribute schemas

.github/workflows/docker-build-static.yml

@@ -299,6 +299,11 @@ jobs:
 
 
     steps:
+       - name: Checkout scripts
+         uses: actions/checkout@v3.5.3
+         with:
+           sparse-checkout: 'scripts'
+
        - name: Download LLDAP artifacts
          uses: actions/download-artifact@v3
          with:
@@ -347,9 +352,7 @@ jobs:
 
        - name: Export and Converting to Postgress
          run: |
-              curl -L https://raw.githubusercontent.com/lldap/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
-              chmod +x ./helper.sh
-              ./helper.sh | sqlite3 ./users.db > ./dump.sql
+              bash ./scripts/sqlite_dump_commands.sh | sqlite3 ./users.db > ./dump.sql
               sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" -e ":a; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),1([^']*\);)$/\1,true\2/; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),0([^']*\);)$/\1,false\2/; ta" -e '1s/^/BEGIN;\n/' -e '$aCOMMIT;' ./dump.sql
 
        - name: Create schema on postgres
@@ -359,14 +362,13 @@ jobs:
        - name: Copy converted db to postgress and import
          run: |
               docker cp ./dump.sql postgresql:/tmp/dump.sql
-              docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql"
+              docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql" | tee import.log
               rm ./dump.sql
+              ! grep ERROR import.log > /dev/null
 
        - name: Export and Converting to mariadb
          run: |
-              curl -L https://raw.githubusercontent.com/lldap/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
-              chmod +x ./helper.sh
-              ./helper.sh | sqlite3 ./users.db > ./dump.sql
+              bash ./scripts/sqlite_dump_commands.sh | sqlite3 ./users.db > ./dump.sql
               cp ./dump.sql ./dump-no-sed.sql
               sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
               sed  -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
@@ -377,14 +379,13 @@ jobs:
        - name: Copy converted db to mariadb and import
          run: |
               docker cp ./dump.sql mariadb:/tmp/dump.sql
-              docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
+              docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql" | tee import.log
               rm ./dump.sql
+              ! grep ERROR import.log > /dev/null
 
        - name: Export and Converting to mysql
          run: |
-              curl -L https://raw.githubusercontent.com/lldap/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
-              chmod +x ./helper.sh
-              ./helper.sh | sqlite3 ./users.db > ./dump.sql
+              bash ./scripts/sqlite_dump_commands.sh | sqlite3 ./users.db > ./dump.sql
               sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
               sed  -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
 
@@ -394,8 +395,9 @@ jobs:
        - name: Copy converted db to mysql and import
          run: |
               docker cp ./dump.sql mysql:/tmp/dump.sql
-              docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
+              docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql" | tee import.log
               rm ./dump.sql
+              ! grep ERROR import.log > /dev/null
 
        - name: Run lldap with postgres DB and healthcheck again
          run: |

docs/database_migration.md

@@ -20,7 +20,7 @@ LLDAP has a command that will connect to a target database and initialize the
 schema. If running with docker, run the following command to use your active
 instance (this has the benefit of ensuring your container has access):
 
-```
+```sh
 docker exec -it <LLDAP container name> /app/lldap create_schema -d <Target database url>
 ```
 
@@ -34,7 +34,7 @@ databases (SQLite in this example) will give an error if LLDAP is in the middle
 statements. There are various ways to do this, but a simple enough way is filtering a
 whole database dump. This repo contains [a script](/scripts/sqlite_dump_commands.sh) to generate SQLite commands for creating an appropriate dump:
 
-```
+```sh
 ./sqlite_dump_commands.sh | sqlite3 /path/to/lldap/config/users.db > /path/to/dump.sql
 ```
 
@@ -49,8 +49,9 @@ a transaction in case one of the statements fail.
 PostgreSQL uses a different hex string format. The command below should switch SQLite
 format to PostgreSQL format, and wrap it all in a transaction:
 
-```
+```sh
 sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" \
+-e ":a; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),1([^']*\);)$/\1,true\2/; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),0([^']*\);)$/\1,false\2/; ta" \
 -e '1s/^/BEGIN;\n/' \
 -e '$aCOMMIT;' /path/to/dump.sql
 ```
@@ -58,11 +59,11 @@ sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" \
 ### To MySQL
 
 MySQL mostly cooperates, but it gets some errors if you don't escape the `groups` table. It also uses
-backticks to escape table name instead of quotes.  Run the
+backticks to escape table name instead of quotes. Run the
 following command to wrap all table names in backticks for good measure, and wrap the inserts in
 a transaction:
 
-```
+```sh
 sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
 -e '1s/^/START TRANSACTION;\n/' \
 -e '$aCOMMIT;' \
@@ -74,7 +75,7 @@ sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
 While MariaDB is supposed to be identical to MySQL, it doesn't support timezone offsets on DATETIME
 strings. Use the following command to remove those and perform the additional MySQL sanitization:
 
-```
+```sh
 sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \
 -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
 -e '1s/^/START TRANSACTION;\n/' \

scripts/sqlite_dump_commands.sh

@@ -1,9 +1,15 @@
 #! /bin/bash
 
-tables=("users" "groups" "memberships" "jwt_refresh_storage" "jwt_storage" "password_reset_tokens" "group_attribute_schema" "group_attributes" "user_attribute_schema" "user_attributes")
+tables=("users" "groups" "memberships" "jwt_refresh_storage" "jwt_storage" "password_reset_tokens" "group_attribute_schema" "group_attributes")
 echo ".header on"
 
 for table in ${tables[@]}; do
     echo ".mode insert $table"
     echo "select * from $table;"
 done
+
+echo ".mode insert user_attribute_schema"
+echo "select * from user_attribute_schema where user_attribute_schema_name not in ('first_name', 'last_name', 'avatar');"
+
+echo ".mode insert user_attributes"
+echo "select * from user_attributes;"