From 9e179d5e061a3b393efe1404520a775f7b84aa30 Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Mon, 5 Sep 2022 10:40:29 +0300 Subject: [PATCH 1/7] Added error handling for incorrectly formatted google creds --- webapp/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/webapp/utils.py b/webapp/utils.py index d79446c..759f8a4 100644 --- a/webapp/utils.py +++ b/webapp/utils.py @@ -157,3 +157,16 @@ def add_to_mailinglist(email: str): ) send_email(to, subject, body) + except ValueError as err: + logging.exception("Formatting of google credentials is incorrect") + + if DEPLOY_ENV == "production": + to = "ilari.ojakorpi@sahkoinsinoorikilta.fi" + subject = "Web error: Failed adding to google groups" + body = "Google credential formatted incorretly\nEmail that was not added: {}\n\nAdd user manually to jäsenet groups.".format( + email + ) + + send_email(to, subject, body) + + From 5575186570f4a2cc0bd6ccfe82a0e289d8d333d3 Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Mon, 5 Sep 2022 12:03:31 +0300 Subject: [PATCH 2/7] lint --- webapp/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/webapp/utils.py b/webapp/utils.py index 759f8a4..328874c 100644 --- a/webapp/utils.py +++ b/webapp/utils.py @@ -168,5 +168,3 @@ def add_to_mailinglist(email: str): ) send_email(to, subject, body) - - From 37a9750d4df51c64a82487f4f8c650a523f51cab Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Mon, 5 Sep 2022 13:38:26 +0300 Subject: [PATCH 3/7] Add group key and dev secrets to stack compose --- production_entrypoint.sh | 3 +++ stack-compose-dev.yml | 8 ++++++++ stack-compose.yml | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/production_entrypoint.sh b/production_entrypoint.sh index 1415f92..afd588c 100755 --- a/production_entrypoint.sh +++ b/production_entrypoint.sh @@ -13,6 +13,9 @@ fi if test -f "$GOOGLE_CREDS_FILE"; then export GOOGLE_CREDS=$(cat $GOOGLE_CREDS_FILE) fi +if test -f "$GROUP_KEY_FILE"; then + export GROUP_KEY=$(cat $GROUP_KEY_FILE) +fi # Collect static files echo "Collect static files" diff --git a/stack-compose-dev.yml b/stack-compose-dev.yml index d92b95f..ae075f6 100644 --- a/stack-compose-dev.yml +++ b/stack-compose-dev.yml @@ -29,15 +29,23 @@ services: - FRONTEND_URL=dev.sahkoinsinoorikilta.fi - DEBUG=True - EMAIL_API_KEY_FILE=/run/secrets/DJANGO_EMAIL_API_KEY + - GOOGLE_CREDS_FILE=/run/secrets/BACKEND_GOOGLE_CREDS + - GROUP_KEY_FILE=/run/secrets/BACKEND_GROUP_KEY - DB_HOST=db - DB_PORT=5432 secrets: - DJANGO_EMAIL_API_KEY + - BACKEND_GOOGLE_CREDS + - BACKEND_GROUP_KEY secrets: DJANGO_EMAIL_API_KEY: external: true + BACKEND_GOOGLE_CREDS: + external: true + BACKEND_GROUP_KEY: + external: true volumes: dbdata: diff --git a/stack-compose.yml b/stack-compose.yml index d780df1..7f1bfc6 100644 --- a/stack-compose.yml +++ b/stack-compose.yml @@ -35,12 +35,14 @@ services: - DB_PASSWD_FILE=/run/secrets/BACKEND_DB_PASSWD - EMAIL_API_KEY_FILE=/run/secrets/BACKEND_EMAIL_API_KEY - GOOGLE_CREDS_FILE=/run/secrets/BACKEND_GOOGLE_CREDS + - GROUP_KEY_FILE=/run/secrets/BACKEND_GROUP_KEY secrets: - BACKEND_SECRET_KEY - BACKEND_DB_PASSWD - BACKEND_EMAIL_API_KEY - BACKEND_GOOGLE_CREDS + - BACKEND_GROUP_KEY secrets: BACKEND_SECRET_KEY: @@ -51,3 +53,5 @@ secrets: external: true BACKEND_GOOGLE_CREDS: external: true + BACKEND_GROUP_KEY: + external: true From f0179c1840dff72332d9ed1c5d03be572fc7e75a Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Mon, 12 Sep 2022 22:38:18 +0300 Subject: [PATCH 4/7] Change google creds format. Ugly but works. --- production_entrypoint.sh | 16 ++++++++++++++-- sikweb/settings.py | 16 ++++++++++++---- stack-compose-dev.yml | 22 +++++++++++++++++++--- stack-compose.yml | 22 +++++++++++++++++++--- webapp/utils.py | 12 ++++++++---- 5 files changed, 72 insertions(+), 16 deletions(-) diff --git a/production_entrypoint.sh b/production_entrypoint.sh index afd588c..fd52900 100755 --- a/production_entrypoint.sh +++ b/production_entrypoint.sh @@ -10,8 +10,20 @@ fi if test -f "$DB_PASSWD_FILE"; then export DB_PASSWD=$(cat $DB_PASSWD_FILE) fi -if test -f "$GOOGLE_CREDS_FILE"; then - export GOOGLE_CREDS=$(cat $GOOGLE_CREDS_FILE) +if test -f "$G_PRIVATE_KEY_ID_FILE"; then + export G_PRIVATE_KEY_ID=$(cat $G_PRIVATE_KEY_ID_FILE) +fi +if test -f "$G_PRIVATE_KEY_FILE"; then + export G_PRIVATE_KEY=$(cat $G_PRIVATE_KEY_FILE) +fi +if test -f "$G_CLIENT_EMAIL_FILE"; then + export G_CLIENT_EMAIL=$(cat $G_CLIENT_EMAIL_FILE) +fi +if test -f "$G_CLIENT_ID_FILE"; then + export G_CLIENT_ID=$(cat $G_CLIENT_ID_FILE) +fi +if test -f "$G_CLIENT_URL_FILE"; then + export G_CLIENT_URL=$(cat $G_CLIENT_URL_FILE) fi if test -f "$GROUP_KEY_FILE"; then export GROUP_KEY=$(cat $GROUP_KEY_FILE) diff --git a/sikweb/settings.py b/sikweb/settings.py index 87a5e69..5580203 100644 --- a/sikweb/settings.py +++ b/sikweb/settings.py @@ -82,11 +82,19 @@ DATABASES = { # Google api settings GROUP_KEY = os.getenv("GROUP_KEY", "") -try: - GOOGLE_CREDS_JSON = json.loads(os.getenv("GOOGLE_CREDS", "{}")) -except: - GOOGLE_CREDS_JSON = json.loads("{}") +GOOGLE_CREDS = { + "type": "service_account", + "project_id": "web2-backend", + "private_key_id": os.getenv("G_PRIVATE_KEY_ID", ""), + "private_key": os.getenv("G_PRIVATE_KEY", ""), + "client_email": os.getenv("G_CLIENT_EMAIL", ""), + "client_id": os.getenv("G_CLIENT_ID", ""), + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": os.getenv("G_CLIENT_URL", ""), +} # JWT authentication SIMPLE_JWT = { diff --git a/stack-compose-dev.yml b/stack-compose-dev.yml index ae075f6..ff3cc4c 100644 --- a/stack-compose-dev.yml +++ b/stack-compose-dev.yml @@ -29,20 +29,36 @@ services: - FRONTEND_URL=dev.sahkoinsinoorikilta.fi - DEBUG=True - EMAIL_API_KEY_FILE=/run/secrets/DJANGO_EMAIL_API_KEY - - GOOGLE_CREDS_FILE=/run/secrets/BACKEND_GOOGLE_CREDS + - G_PRIVATE_KEY_ID_FILE=/run/secrets/BACKEND_G_PRIVATE_KEY_ID + - G_PRIVATE_KEY_FILE=/run/secrets/BACKEND_G_PRIVATE_KEY + - G_CLIENT_EMAIL_FILE=/run/secrets/BACKEND_G_CLIENT_EMAIL + - G_CLIENT_ID_FILE=/run/secrets/BACKEND_G_CLIENT_ID + - G_CLIENT_URL_FILE=/run/secrets/BACKEND_G_CLIENT_URL - GROUP_KEY_FILE=/run/secrets/BACKEND_GROUP_KEY - DB_HOST=db - DB_PORT=5432 secrets: - DJANGO_EMAIL_API_KEY - - BACKEND_GOOGLE_CREDS + - BACKEND_G_PRIVATE_KEY_ID + - BACKEND_G_PRIVATE_KEY + - BACKEND_G_CLIENT_EMAIL + - BACKEND_G_CLIENT_ID + - BACKEND_G_CLIENT_URL - BACKEND_GROUP_KEY secrets: DJANGO_EMAIL_API_KEY: external: true - BACKEND_GOOGLE_CREDS: + BACKEND_G_PRIVATE_KEY_ID: + external: true + BACKEND_G_PRIVATE_KEY: + external: true + BACKEND_G_CLIENT_EMAIL: + external: true + BACKEND_G_CLIENT_ID: + external: true + BACKEND_G_CLIENT_URL: external: true BACKEND_GROUP_KEY: external: true diff --git a/stack-compose.yml b/stack-compose.yml index 7f1bfc6..a336faf 100644 --- a/stack-compose.yml +++ b/stack-compose.yml @@ -34,14 +34,22 @@ services: - SECRET_KEY_FILE=/run/secrets/BACKEND_SECRET_KEY - DB_PASSWD_FILE=/run/secrets/BACKEND_DB_PASSWD - EMAIL_API_KEY_FILE=/run/secrets/BACKEND_EMAIL_API_KEY - - GOOGLE_CREDS_FILE=/run/secrets/BACKEND_GOOGLE_CREDS + - G_PRIVATE_KEY_ID_FILE=/run/secrets/BACKEND_G_PRIVATE_KEY_ID + - G_PRIVATE_KEY_FILE=/run/secrets/BACKEND_G_PRIVATE_KEY + - G_CLIENT_EMAIL_FILE=/run/secrets/BACKEND_G_CLIENT_EMAIL + - G_CLIENT_ID_FILE=/run/secrets/BACKEND_G_CLIENT_ID + - G_CLIENT_URL_FILE=/run/secrets/BACKEND_G_CLIENT_URL - GROUP_KEY_FILE=/run/secrets/BACKEND_GROUP_KEY secrets: - BACKEND_SECRET_KEY - BACKEND_DB_PASSWD - BACKEND_EMAIL_API_KEY - - BACKEND_GOOGLE_CREDS + - BACKEND_G_PRIVATE_KEY_ID + - BACKEND_G_PRIVATE_KEY + - BACKEND_G_CLIENT_EMAIL + - BACKEND_G_CLIENT_ID + - BACKEND_G_CLIENT_URL - BACKEND_GROUP_KEY secrets: @@ -51,7 +59,15 @@ secrets: external: true BACKEND_EMAIL_API_KEY: external: true - BACKEND_GOOGLE_CREDS: + BACKEND_G_PRIVATE_KEY_ID: + external: true + BACKEND_G_PRIVATE_KEY: + external: true + BACKEND_G_CLIENT_EMAIL: + external: true + BACKEND_G_CLIENT_ID: + external: true + BACKEND_G_CLIENT_URL: external: true BACKEND_GROUP_KEY: external: true diff --git a/webapp/utils.py b/webapp/utils.py index 328874c..06cadb2 100644 --- a/webapp/utils.py +++ b/webapp/utils.py @@ -25,7 +25,7 @@ from sikweb.settings import ( DEFAULT_EMAIL_FROM_ADDR, ENABLE_AUTOMATIC_EMAILS, GROUP_KEY, - GOOGLE_CREDS_JSON, + GOOGLE_CREDS, ) from datetime import timedelta @@ -136,13 +136,17 @@ def add_to_mailinglist(email: str): # create credentials, with subject is used to impersonate admin account # jas_manager has groups editor rights in google admin credentials = service_account.Credentials.from_service_account_info( - info=GOOGLE_CREDS_JSON, scopes=SCOPES + info=GOOGLE_CREDS, scopes=SCOPES ).with_subject("jas_manager@sahkoinsinoorikilta.fi") service = build("admin", "directory_v1", credentials=credentials) service.members().insert(groupKey=GROUP_KEY, body={"email": email}).execute() except HttpError as err: # Already in list, do nothing + print() + print(err) + print() + if err.status_code == 409: pass else: @@ -157,7 +161,7 @@ def add_to_mailinglist(email: str): ) send_email(to, subject, body) - except ValueError as err: + """ except ValueError as err: logging.exception("Formatting of google credentials is incorrect") if DEPLOY_ENV == "production": @@ -167,4 +171,4 @@ def add_to_mailinglist(email: str): email ) - send_email(to, subject, body) + send_email(to, subject, body) """ From 2e0fad4bb2b0fab8a3c1dedf2c53c426677eaca0 Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Mon, 12 Sep 2022 22:42:01 +0300 Subject: [PATCH 5/7] Use POSIX format for source --- .husky/pre-commit | 2 +- .husky/pre-push | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index 2867590..32d0bfb 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -4,7 +4,7 @@ PURPLE='\033[0;35m' NC='\033[0m' # No Color -source "${VIRTUAL_ENV}/bin/activate" +. "${VIRTUAL_ENV}/bin/activate" if [ $? -ne 0 ] then diff --git a/.husky/pre-push b/.husky/pre-push index 07e64c7..eb484ac 100755 --- a/.husky/pre-push +++ b/.husky/pre-push @@ -4,7 +4,7 @@ PURPLE='\033[0;35m' NC='\033[0m' # No Color -source "${VIRTUAL_ENV}/bin/activate" +. "${VIRTUAL_ENV}/bin/activate" if [ $? -ne 0 ] then From 19975877cb5bf3b0b1b4e60c974c0485344433ee Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Tue, 13 Sep 2022 09:59:55 +0300 Subject: [PATCH 6/7] Remove debug stuff --- webapp/utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/webapp/utils.py b/webapp/utils.py index 06cadb2..19c2f6c 100644 --- a/webapp/utils.py +++ b/webapp/utils.py @@ -143,10 +143,6 @@ def add_to_mailinglist(email: str): service.members().insert(groupKey=GROUP_KEY, body={"email": email}).execute() except HttpError as err: # Already in list, do nothing - print() - print(err) - print() - if err.status_code == 409: pass else: @@ -161,7 +157,7 @@ def add_to_mailinglist(email: str): ) send_email(to, subject, body) - """ except ValueError as err: + except ValueError as err: logging.exception("Formatting of google credentials is incorrect") if DEPLOY_ENV == "production": @@ -171,4 +167,4 @@ def add_to_mailinglist(email: str): email ) - send_email(to, subject, body) """ + send_email(to, subject, body) From a8923b63d64afcf422af083d33d02a54deb80f96 Mon Sep 17 00:00:00 2001 From: Ojakoo Date: Tue, 13 Sep 2022 13:31:58 +0300 Subject: [PATCH 7/7] wrap private key --- production_entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production_entrypoint.sh b/production_entrypoint.sh index fd52900..0efc365 100755 --- a/production_entrypoint.sh +++ b/production_entrypoint.sh @@ -14,7 +14,7 @@ if test -f "$G_PRIVATE_KEY_ID_FILE"; then export G_PRIVATE_KEY_ID=$(cat $G_PRIVATE_KEY_ID_FILE) fi if test -f "$G_PRIVATE_KEY_FILE"; then - export G_PRIVATE_KEY=$(cat $G_PRIVATE_KEY_FILE) + export G_PRIVATE_KEY="$(cat $G_PRIVATE_KEY_FILE)" fi if test -f "$G_CLIENT_EMAIL_FILE"; then export G_CLIENT_EMAIL=$(cat $G_CLIENT_EMAIL_FILE)