Store profile pictures in S3-compatible storage

.env.example

@@ -20,3 +20,15 @@ REDIS_URI=redis://localhost:6379/0
 # Generate a random secret key using `openssl rand -base64 32`
 SECRET_KEY='oefwcb5EeYdvOinNIrJlofhVAAtbSardsFGNPQHjwCA='
 TOKEN_VALIDITY_SECS=3600
+
+
+# Place a S3-compatible object store config here. This example is for MinIO
+STORAGE_BASE_URL=http://localhost:9000/user-content
+S3_HOST=localhost:9000
+S3_ACCESS_KEY=root
+S3_SECRET_KEY=snowflake
+S3_BUCKET=user-content
+
+
+# Enables minIO browser, disable on production
+MINIO_BROWSER=off

.github/workflows/ci.yaml

@@ -19,6 +19,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip pipenv
           pipenv install --deploy --system --dev
+          cp .env.example .env
       - name: Test
         run: |
           make clean all

Pipfile

@@ -36,6 +36,7 @@ flask-redis = "*"
 werkzeug = "*"
 alembic = "*"
 flask-migrate = "*"
+minio = "*"
 
 [requires]
 python_version = "3.9"

README.md

@@ -18,7 +18,13 @@ Snowflake
    leave it running.
 3. Active a Pipenv shell using `pipenv shell`
 4. Install dependencies `pipenv install && npm install`
-5. Run migrations using `python migration.py`
-6. Start asset pipeline with `npm start` and leave it running.
-7. Start the application with `flask run`
-8. Run tests with `make`
+5. Run migrations using `flask db upgrade`
+6. Configure included MinIO with `minio/stable/mc` (`brew install minio/stable/mc`)
+   ```shell
+   mc alias set local http://localhost:9000 root snowflake
+   mc mb local/user-content
+   mc policy set-json scripts/minio-policy.json local/user-content
+   ```
+7. Start asset pipeline with `npm start` and leave it running.
+8. Start the application with `flask run`
+9. Run tests with `make`

docker-compose.yml

@@ -2,11 +2,25 @@ version: '3'
 services:
   db:
     image: postgres:13
-    env_file:
-      - .env
+    environment:
+      - POSTGRES_PASSWORD:${POSTGRES_PASSWORD}
     ports:
       - 5432:5432
   redis:
     image: redis:6
     ports:
       - 6379:6379
+  minio:
+    image: 'minio/minio:latest'
+    ports:
+      - 9000:9000
+    command: 'minio server /srv'
+    environment:
+      - MINIO_ROOT_USER=${S3_ACCESS_KEY}
+      - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY}
+      - MINIO_BROWSER=${MINIO_BROWSER}
+    volumes:
+      - minio_data:/srv
+
+volumes:
+  minio_data:

scripts/minio-policy.json

@@ -0,0 +1,34 @@
+{
+  "Statement": [
+    {
+      "Action": [
+        "s3:GetBucketLocation",
+        "s3:ListBucket"
+      ],
+      "Effect": "Deny",
+      "Principal": {
+        "AWS": [
+          "*"
+        ]
+      },
+      "Resource": [
+        "arn:aws:s3:::user-content"
+      ]
+    },
+    {
+      "Action": [
+        "s3:GetObject"
+      ],
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": [
+          "*"
+        ]
+      },
+      "Resource": [
+        "arn:aws:s3:::user-content/*"
+      ]
+    }
+  ],
+  "Version": "2012-10-17"
+}

snowflake/controllers/register.py

@@ -1,28 +1,42 @@
+import secrets
+
 from flask import Blueprint, request, session, redirect, url_for, render_template
 from flask_login import login_user
 
 from snowflake import db
 from snowflake.forms import RegistrationForm
 from snowflake.models import User
+from snowflake.services import file_system
 
 blueprint = Blueprint('register', __name__)
 
 
+def profile_picture_url():
+    return f'user-profile-pictures/{secrets.token_urlsafe(32)}.jpg'
+
+
 @blueprint.route('/', methods=['GET', 'POST'])
 def register():
     form = RegistrationForm()
 
     if request.method == 'POST' and form.validate():
         unique_id = session['unique_id']
         users_email = session['users_email']
-        picture = session['picture']
+        picture_url = session['picture']
         full_name = session['users_name']
         username = users_email.split("@")[0]
 
-        user = User(id=unique_id, email=users_email, name=full_name, profile_pic=picture,
+        user = User(id=unique_id, email=users_email, name=full_name, profile_pic=None,
                     team_name=form.team_name.data, designation=form.designation.data,
                     username=username)
 
+        with db.transaction():
+            db.persist(user)
+
+        uploaded_url = file_system.put_remote_object(profile_picture_url(), picture_url)
+
+        user.profile_pic = uploaded_url
+
         with db.transaction():
             db.persist(user)
 

snowflake/services/file_system.py

@@ -0,0 +1,43 @@
+import requests
+from minio import Minio
+
+from snowflake import settings
+
+STORAGE_BASE_URL = settings.storage_base_url()
+S3_SETTINGS = settings.s3_settings()
+USER_AGENT = f'Snowflake/1.0 (+{settings.base_url()})'
+
+minio = Minio(
+    S3_SETTINGS['HOST'],
+    access_key=S3_SETTINGS['ACCESS_KEY'],
+    secret_key=S3_SETTINGS['SECRET_KEY'],
+    secure=S3_SETTINGS['SECURE']
+)
+BUCKET_NAME = S3_SETTINGS['BUCKET']
+
+
+def make_http_request(url):
+    return requests.get(url, headers={
+        'User-Agent': USER_AGENT
+    }, allow_redirects=True, stream=True)
+
+
+def try_parse_len(string):
+    try:
+        return int(string)
+    except ValueError:
+        return -1
+
+
+def put_remote_object(key, url):
+    response = make_http_request(url)
+
+    if response.status_code != 200:
+        raise ValueError(f'Server returned status {response.status_code}')
+
+    response.raw.decode_content = True
+    content_length = try_parse_len(response.headers.get('Content-Length', '-1'))
+    content_type = response.headers.get('Content-Type', 'application/octet-stream')
+
+    minio.put_object(BUCKET_NAME, key, response.raw, content_length, content_type=content_type)
+    return STORAGE_BASE_URL + '/' + key

snowflake/settings.py

@@ -18,6 +18,10 @@ def database_uri():
     return os.getenv('DATABASE_URI')
 
 
+def base_url():
+    return os.getenv('BASE_URL', 'http://127.0.0.1:5000')
+
+
 def init_app(app: Flask):
     app.secret_key = os.getenvb(b"SECRET_KEY") or os.urandom(24)
 
@@ -28,5 +32,19 @@ def init_app(app: Flask):
         GOOGLE_CLIENT_ID=os.getenv("GOOGLE_CLIENT_ID"),
         GOOGLE_CLIENT_SECRET=os.getenv("GOOGLE_CLIENT_SECRET"),
         GOOGLE_PROVIDER_CONFIG=requests.get(GOOGLE_DISCOVERY_URL).json(),
-        BASE_URL=os.getenv('BASE_URL', 'http://127.0.0.1:5000'),
+        BASE_URL=base_url(),
     )
+
+
+def storage_base_url():
+    return os.getenv('STORAGE_BASE_URL', base_url()).rstrip('/')
+
+
+def s3_settings():
+    return {
+        'BUCKET': os.getenv('S3_BUCKET'),
+        'HOST': os.getenv('S3_HOST'),
+        'ACCESS_KEY': os.getenv('S3_ACCESS_KEY'),
+        'SECRET_KEY': os.getenv('S3_SECRET_KEY'),
+        'SECURE': bool(os.getenv('S3_SECURE', 'False')),
+    }

snowflake/templates/home.html

@@ -184,7 +184,7 @@ <h3 class="is-size-5 mb-0 text-center">
                 <div style="position: sticky; top:4.5rem">
                     <div class="box block">
                         <div class="header pb-0 p-4 has-text-centered">
-                            <figure class="avatar image mb-3 is-128x128">
+                            <figure class="avatar image mb-3 is-64x64">
                                 <img alt="Profile picture of {{ user.name }}"
                                      title="{{ user.name }}" class="is-rounded" src="{{ user.profile_pic }}">
                             </figure>