Merge branch 'release/1.0'

.gitlab-ci.yml

@@ -97,7 +97,7 @@ black:
   variables:
     GIT_STRATEGY: fetch
   before_script:
-    - pip install black
+    - pip install black==19.10b0
   script:
     - black --check --diff api/
 

CHANGELOG

@@ -10,6 +10,149 @@ This changelog is viewable on the web at https://docs.funkwhale.audio/changelog.
 
 .. towncrier
 
+1.0 (2020-09-09)
+----------------
+
+Upgrade instructions are available at
+https://docs.funkwhale.audio/index.html
+
+
+Dropped python 3.5 support [manual action required, non-docker only]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With Funkwhale 1.0, we're dropping support for Python 3.5. Before upgrading,
+ensure ``python3 --version`` returns ``3.6`` or higher.
+
+If it returns ``3.6`` or higher, you have nothing to do.
+
+If it returns ``3.5``, you will need to upgrade your Python version/Host, then recreate your virtual environment::
+
+    rm -rf /srv/funkwhale/virtualenv
+    python3 -m venv /srv/funkwhale/virtualenv
+
+
+Increased quality of JPEG thumbnails [manual action required]
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Default quality for JPEG thumbnails was increased from 70 to 95, as 70 was producing visible artifacts in resized images.
+
+Because of this change, existing thumbnails will not load, and you will need to:
+
+1. delete the ``__sized__`` directory in your ``MEDIA_ROOT`` directory
+2. run ``python manage.py fw media generate-thumbnails`` to regenerate thumbnails with the enhanced quality
+
+If you don't want to regenerate thumbnails, you can keep the old ones by adding ``THUMBNAIL_JPEG_RESIZE_QUALITY=70`` to your .env file.
+
+Small API breaking change in ``/api/v1/libraries``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To allow easier crawling of public libraries on a pod,we had to make a slight breaking change
+to the behaviour of ``GET /api/v1/libraries``.
+
+Before, it returned only libraries owned by the current user.
+
+Now, it returns all the accessible libraries (including ones from other users and pods).
+
+If you are consuming the API via a third-party client and need to retrieve your libraries,
+use the ``scope`` parameter, like this: ``GET /api/v1/libraries?scope=me``
+
+API breaking change in ``/api/v1/albums``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To increase performance, querying ``/api/v1/albums`` doesn't return album tracks anymore. This caused
+some performance issues, especially as some albums and series have dozens or even hundreds of tracks.
+
+If you want to retrieve tracks for an album, you can query ``/api/v1/tracks/?album=<albumid>``.
+
+JWT deprecation
+^^^^^^^^^^^^^^^
+
+API Authentication using JWT is deprecated and will be removed in Funkwhale 1.0. Please use OAuth or application tokens
+and refer to our API documentation at https://docs.funkwhale.audio/swagger/ for guidance.
+
+Full list of changes
+^^^^^^^^^^^^^^^^^^^^
+
+Features:
+
+- Allow users to hide compilation artists on the artist search page (#1053)
+- Can now launch server import from the UI (#1105)
+- Dedicated, advanced search page (#370)
+- Persist theme and language settings accross sessions (#996)
+
+
+Enhancements:
+
+- Add support for unauthenticated users hitting the logout page
+- Added support for Licence Art Libre (#1088)
+- Broadcast/handle rejected follows (#858)
+- Confirm email without requiring the user to validate the form manually (#407)
+- Display channel and track downloads count (#1178)
+- Do not include tracks in album API representation (#1102)
+- Dropped python 3.5 support. Python 3.6 is the minimum required version (#1099)
+- Improved keyboard accessibility (#1125)
+- Improved naming of pages for accessibility (#1127)
+- Improved shuffle behaviour (#1190)
+- Increased quality of JPEG thumbnails
+- Lock focus in modals to improve accessibility (#1128)
+- More consistent search UX on /albums, /artists, /radios and /playlists (#1131)
+- Play button now replace current queue instead of appending to it (#1083)
+- Set proper lang attribute on HTML document (#1130)
+- Use semantic headers for accessibility (#1121)
+- Users can now update their email address (#292)
+- [plugin, scrobbler] Use last.fm API v2 for scrobbling if API key and secret are provided
+- Added a new, large thumbnail size for cover images (#1205
+- Enforce authentication when viewing remote channels, profiles and libraries (#1210)
+
+
+
+Bugfixes:
+
+- Fix broken media support detection (#1180)
+- Fix layout issue with playbar on landscape tablets (#1144)
+- Fix random radio so that podcast content is not picked up (#1140)
+- Fixed an issue with search pages where results would not appear after navigating to another page
+- Fixed crash with negative track position in file tags (#1193)
+- Handle access errors scanning directories when importing files
+- Make channel card updated times more humanly readable, add internationalization (#1089)
+- Ensure search page reloads if another search is submitted in the sidebar (#1197)
+- Fixed "scope=subscribed" on albums, artists, uploads and libraries API (#1217)
+- Fixed broken federation with pods using allow-listing (#1999)
+- Fixed broken search when using (, " or & chars (#1196)
+- Fixed domains table hidden controls when no domains are found (#1198)
+
+
+Documentation:
+
+- Simplify Docker mono-container installation and upgrade documentation
+
+
+Contributors to this release (translation, development, documentation, reviews, design, testing, third-party projects):
+
+- Agate
+- Andy Craze
+- anonymous
+- appzer0
+- Arne
+- Bheesham Persaud
+- Ciarán Ainsworth
+- Creak
+- Daniele Lira Mereb
+- dulz
+- Francesc Galí
+- ghose
+- mekind
+- Puri
+- Quentin PAGÈS
+- Raphaël Ventura
+- Simon Arlott
+- Slimane Selyan Amiri
+- Stefano Pigozzi
+- Sébastien de Melo
+- vicdorke
+- Xosé M
+
+
 0.21.2 (2020-07-27)
 -------------------
 
@@ -223,7 +366,8 @@ All user-related commands are available under the ``python manage.py fw users``
 Please refer to the `Admin documentation <https://docs.funkwhale.audio/admin/commands.html#user-management>`_ for
 more information and instructions.
 
-Progressive web app [Manual action suggested, non-docker only]
+Progressive web app [Manual action sugFull list of changes
+^^^^^^^^^^^^^^^^^^^^gested, non-docker only]
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 We've made Funkwhale's Web UI a Progressive Web Application (PWA), in order to improve the user experience

CONTRIBUTING.rst

@@ -704,6 +704,21 @@ Views: you can find some readable views tests in file: ``api/tests/users/test_vi
 Contributing to the front-end
 -----------------------------
 
+Styles and themes
+^^^^^^^^^^^^^^^^^
+
+Our UI framework is Fomantic UI (https://fomantic-ui.com/), and Funkwhale's custom styles are written in SCSS. All the styles are configured in ``front/src/styles/_main.scss``,
+including imporing of Fomantic UI styles and components.
+
+We're applying several changes on top of the Fomantic CSS files, before they are imported:
+
+1. Many hardcoded color values are replaced by CSS vars: e.g ``color: orange`` is replaced by ``color: var(--vibrant-color)``. This makes theming way easier.
+2. Unused components variations and icons are stripped from the source files, in order to reduce the final size of our CSS files
+
+This changes are applied automatically when running ``yarn install``, through a ``postinstall`` hook. Internally, ``front/scripts/fix-fomantic-css.py`` is called
+and handle both kind of modifications. Please refer to this script if you need to use new icons to the project, or restore some components variations that
+were stripped in order to use them.
+
 Running tests
 ^^^^^^^^^^^^^
 

README.rst

@@ -28,6 +28,16 @@ Contribute
 Contribution guidelines as well as development installation instructions
 are outlined in `CONTRIBUTING <CONTRIBUTING.rst>`_.
 
+Security issues and vulnerabilities
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you found a vulnerability in Funkwhale, please report it on our Gitlab instance at `https://dev.funkwhale.audio/funkwhale/funkwhale/-/issues`_, ensuring
+you have checked the ``This issue is confidential and should only be visible to team members with at least Reporter access.
+`` box. 
+
+This will ensure only maintainers and developpers have access to the vulnerability. Thank you for your help!
+
+
 Translate
 ^^^^^^^^^
 

api/config/api_urls.py

 from django.conf.urls import include, url
-from dynamic_preferences.api.viewsets import GlobalPreferencesViewSet
 from rest_framework import routers
 from rest_framework.urlpatterns import format_suffix_patterns
 
@@ -14,22 +13,20 @@ from funkwhale_api.tags import views as tags_views
 from funkwhale_api.users import jwt_views
 
 router = common_routers.OptionalSlashRouter()
-router.register(r"settings", GlobalPreferencesViewSet, basename="settings")
 router.register(r"activity", activity_views.ActivityViewSet, "activity")
 router.register(r"tags", tags_views.TagViewSet, "tags")
+router.register(r"plugins", common_views.PluginViewSet, "plugins")
 router.register(r"tracks", views.TrackViewSet, "tracks")
 router.register(r"uploads", views.UploadViewSet, "uploads")
 router.register(r"libraries", views.LibraryViewSet, "libraries")
 router.register(r"listen", views.ListenViewSet, "listen")
+router.register(r"stream", views.StreamViewSet, "stream")
 router.register(r"artists", views.ArtistViewSet, "artists")
 router.register(r"channels", audio_views.ChannelViewSet, "channels")
 router.register(r"subscriptions", audio_views.SubscriptionsViewSet, "subscriptions")
 router.register(r"albums", views.AlbumViewSet, "albums")
 router.register(r"licenses", views.LicenseViewSet, "licenses")
 router.register(r"playlists", playlists_views.PlaylistViewSet, "playlists")
-router.register(
-    r"playlist-tracks", playlists_views.PlaylistTrackViewSet, "playlist-tracks"
-)
 router.register(r"mutations", common_views.MutationViewSet, "mutations")
 router.register(r"attachments", common_views.AttachmentViewSet, "attachments")
 v1_patterns = router.urls
@@ -77,9 +74,11 @@ v1_patterns += [
         r"^history/",
         include(("funkwhale_api.history.urls", "history"), namespace="history"),
     ),
+    url(r"^", include(("funkwhale_api.users.api_urls", "users"), namespace="users"),),
+    # XXX: remove if Funkwhale 1.1
     url(
         r"^users/",
-        include(("funkwhale_api.users.api_urls", "users"), namespace="users"),
+        include(("funkwhale_api.users.api_urls", "users"), namespace="users-nested"),
     ),
     url(
         r"^oauth/",

api/config/plugins.py

+import copy
+import logging
+import os
+import subprocess
+import sys
+
+import persisting_theory
+from django.core.cache import cache
+from django.db.models import Q
+
+from rest_framework import serializers
+
+logger = logging.getLogger("plugins")
+
+
+class Startup(persisting_theory.Registry):
+    look_into = "persisting_theory"
+
+
+class Ready(persisting_theory.Registry):
+    look_into = "persisting_theory"
+
+
+startup = Startup()
+ready = Ready()
+
+_plugins = {}
+_filters = {}
+_hooks = {}
+
+
+class PluginCache(object):
+    def __init__(self, prefix):
+        self.prefix = prefix
+
+    def get(self, key, default=None):
+        key = ":".join([self.prefix, key])
+        return cache.get(key, default)
+
+    def set(self, key, value, duration=None):
+        key = ":".join([self.prefix, key])
+        return cache.set(key, value, duration)
+
+
+def get_plugin_config(
+    name,
+    user=False,
+    source=False,
+    registry=_plugins,
+    conf={},
+    settings={},
+    description=None,
+    version=None,
+    label=None,
+    homepage=None,
+):
+    conf = {
+        "name": name,
+        "label": label or name,
+        "logger": logger,
+        # conf is for dynamic settings
+        "conf": conf,
+        # settings is for settings hardcoded in .env
+        "settings": settings,
+        "user": True if source else user,
+        # source plugins are plugins that provide audio content
+        "source": source,
+        "description": description,
+        "version": version,
+        "cache": PluginCache(name),
+        "homepage": homepage,
+    }
+    registry[name] = conf
+    return conf
+
+
+def load_settings(name, settings):
+    from django.conf import settings as django_settings
+
+    mapping = {
+        "boolean": django_settings.ENV.bool,
+        "text": django_settings.ENV,
+    }
+    values = {}
+    prefix = "FUNKWHALE_PLUGIN_{}".format(name.upper())
+    for s in settings:
+        key = "_".join([prefix, s["name"].upper()])
+        value = mapping[s["type"]](key, default=s.get("default", None))
+        values[s["name"]] = value
+
+    logger.debug("Plugin %s running with settings %s", name, values)
+    return values
+
+
+def get_session():
+    from funkwhale_api.common import session
+
+    return session.get_session()
+
+
+def register_filter(name, plugin_config, registry=_filters):
+    def decorator(func):
+        handlers = registry.setdefault(name, [])
+
+        def inner(*args, **kwargs):
+            plugin_config["logger"].debug("Calling filter for %s", name)
+            rval = func(*args, **kwargs)
+            return rval
+
+        handlers.append((plugin_config["name"], inner))
+        return inner
+
+    return decorator
+
+
+def register_hook(name, plugin_config, registry=_hooks):
+    def decorator(func):
+        handlers = registry.setdefault(name, [])
+
+        def inner(*args, **kwargs):
+            plugin_config["logger"].debug("Calling hook for %s", name)
+            func(*args, **kwargs)
+
+        handlers.append((plugin_config["name"], inner))
+        return inner
+
+    return decorator
+
+
+class Skip(Exception):
+    pass
+
+
+def trigger_filter(name, value, enabled=False, **kwargs):
+    """
+    Call filters registered for "name" with the given
+    args and kwargs.
+
+    Return the value (that could be modified by handlers)
+    """
+    logger.debug("Calling handlers for filter %s", name)
+    registry = kwargs.pop("registry", _filters)
+    confs = kwargs.pop("confs", {})
+    for plugin_name, handler in registry.get(name, []):
+        if not enabled and confs.get(plugin_name, {}).get("enabled") is False:
+            continue
+        try:
+            value = handler(value, conf=confs.get(plugin_name, {}), **kwargs)
+        except Skip:
+            pass
+        except Exception as e:
+            logger.warn("Plugin %s errored during filter %s: %s", plugin_name, name, e)
+    return value
+
+
+def trigger_hook(name, enabled=False, **kwargs):
+    """
+    Call hooks registered for "name" with the given
+    args and kwargs.
+
+    Returns nothing
+    """
+    logger.debug("Calling handlers for hook %s", name)
+    registry = kwargs.pop("registry", _hooks)
+    confs = kwargs.pop("confs", {})
+    for plugin_name, handler in registry.get(name, []):
+        if not enabled and confs.get(plugin_name, {}).get("enabled") is False:
+            continue
+        try:
+            handler(conf=confs.get(plugin_name, {}).get("conf"), **kwargs)
+        except Skip:
+            pass
+        except Exception as e:
+            logger.warn("Plugin %s errored during hook %s: %s", plugin_name, name, e)
+
+
+def set_conf(name, conf, user=None, registry=_plugins):
+    from funkwhale_api.common import models
+
+    if not registry[name]["conf"] and not registry[name]["source"]:
+        return
+    conf_serializer = get_serializer_from_conf_template(
+        registry[name]["conf"], user=user, source=registry[name]["source"],
+    )(data=conf)
+    conf_serializer.is_valid(raise_exception=True)
+    if "library" in conf_serializer.validated_data:
+        conf_serializer.validated_data["library"] = str(
+            conf_serializer.validated_data["library"]
+        )
+    conf, _ = models.PluginConfiguration.objects.update_or_create(
+        user=user, code=name, defaults={"conf": conf_serializer.validated_data}
+    )
+
+
+def get_confs(user=None):
+    from funkwhale_api.common import models
+
+    qs = models.PluginConfiguration.objects.filter(code__in=list(_plugins.keys()))
+    if user:
+        qs = qs.filter(Q(user=None) | Q(user=user))
+    else:
+        qs = qs.filter(user=None)
+    confs = {
+        v["code"]: {"conf": v["conf"], "enabled": v["enabled"]}
+        for v in qs.values("code", "conf", "enabled")
+    }
+    for p, v in _plugins.items():
+        if p not in confs:
+            confs[p] = {"conf": None, "enabled": False}
+    return confs
+
+
+def get_conf(plugin, user=None):
+    return get_confs(user=user)[plugin]
+
+
+def enable_conf(code, value, user):
+    from funkwhale_api.common import models
+
+    models.PluginConfiguration.objects.update_or_create(
+        code=code, user=user, defaults={"enabled": value}
+    )
+
+
+class LibraryField(serializers.UUIDField):
+    def __init__(self, *args, **kwargs):
+        self.actor = kwargs.pop("actor")
+        super().__init__(*args, **kwargs)
+
+    def to_internal_value(self, v):
+        v = super().to_internal_value(v)
+        if not self.actor.libraries.filter(uuid=v).first():
+            raise serializers.ValidationError("Invalid library id")
+        return v
+
+
+def get_serializer_from_conf_template(conf, source=False, user=None):
+    conf = copy.deepcopy(conf)
+    validators = {f["name"]: f.pop("validator") for f in conf if "validator" in f}
+    mapping = {
+        "url": serializers.URLField,
+        "boolean": serializers.BooleanField,
+        "text": serializers.CharField,
+        "long_text": serializers.CharField,
+        "password": serializers.CharField,
+        "number": serializers.IntegerField,
+    }
+
+    for attr in ["label", "help"]:
+        for c in conf:
+            c.pop(attr, None)
+
+    class Serializer(serializers.Serializer):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            for field_conf in conf:
+                field_kwargs = copy.copy(field_conf)
+                name = field_kwargs.pop("name")
+                self.fields[name] = mapping[field_kwargs.pop("type")](**field_kwargs)
+            if source:
+                self.fields["library"] = LibraryField(actor=user.actor)
+
+    for vname, v in validators.items():
+        setattr(Serializer, "validate_{}".format(vname), v)
+    return Serializer
+
+
+def serialize_plugin(plugin_conf, confs):
+    return {
+        "name": plugin_conf["name"],
+        "label": plugin_conf["label"],
+        "description": plugin_conf.get("description") or None,
+        "user": plugin_conf.get("user", False),
+        "source": plugin_conf.get("source", False),
+        "conf": plugin_conf.get("conf", None),
+        "values": confs.get(plugin_conf["name"], {"conf"}).get("conf"),
+        "enabled": plugin_conf["name"] in confs
+        and confs[plugin_conf["name"]]["enabled"],
+        "homepage": plugin_conf["homepage"],
+    }
+
+
+def install_dependencies(deps):
+    if not deps:
+        return
+    logger.info("Installing plugins dependencies %s", deps)
+    pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
+    subprocess.check_call([pip_path, "install"] + deps)
+
+
+def background_task(name):
+    from funkwhale_api.taskapp import celery
+
+    def decorator(func):
+        return celery.app.task(func, name=name)
+
+    return decorator
+
+
+# HOOKS
+LISTENING_CREATED = "listening_created"
+"""
+Called when a track is being listened
+"""
+SCAN = "scan"
+"""
+
+"""
+# FILTERS
+PLUGINS_DEPENDENCIES = "plugins_dependencies"
+"""
+Called with an empty list, use this filter to append pip dependencies
+to the list for installation.
+"""
+PLUGINS_APPS = "plugins_apps"
+"""
+Called with an empty list, use this filter to append apps to INSTALLED_APPS
+"""
+MIDDLEWARES_BEFORE = "middlewares_before"
+"""
+Called with an empty list, use this filter to prepend middlewares
+to MIDDLEWARE
+"""
+MIDDLEWARES_AFTER = "middlewares_after"
+"""
+Called with an empty list, use this filter to append middlewares
+to MIDDLEWARE
+"""
+URLS = "urls"
+"""
+Called with an empty list, use this filter to register new urls and views
+"""

api/config/routing.py

+from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
-from django.conf.urls import url
 
-from funkwhale_api.common.auth import TokenAuthMiddleware
+from django.conf.urls import url
 from funkwhale_api.instance import consumers
 
 application = ProtocolTypeRouter(
     {
         # Empty for now (http->django views is added by default)
-        "websocket": TokenAuthMiddleware(
+        "websocket": AuthMiddlewareStack(
             URLRouter([url("^api/v1/activity$", consumers.InstanceActivityConsumer)])
         )
     }

api/config/settings/common.py

 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
 
+from collections import OrderedDict
 import datetime
 import logging.config
-import os
 import sys
 
 from urllib.parse import urlsplit
@@ -18,7 +18,7 @@ ROOT_DIR = environ.Path(__file__) - 3  # (/a/b/myfile.py - 3 = /)
 APPS_DIR = ROOT_DIR.path("funkwhale_api")
 
 env = environ.Env()
-
+ENV = env
 LOGLEVEL = env("LOGLEVEL", default="info").upper()
 """
 Default logging level for the Funkwhale processes"""  # pylint: disable=W0105
@@ -46,6 +46,12 @@ logging.config.dictConfig(
                 # required to avoid double logging with root logger
                 "propagate": False,
             },
+            "plugins": {
+                "level": LOGLEVEL,
+                "handlers": ["console"],
+                # required to avoid double logging with root logger
+                "propagate": False,
+            },
             "": {"level": "WARNING", "handlers": ["console"]},
         },
     }
@@ -86,7 +92,31 @@ FUNKWHALE_PLUGINS_PATH = env(
 Path to a directory containing Funkwhale plugins. These will be imported at runtime.
 """
 sys.path.append(FUNKWHALE_PLUGINS_PATH)
+CORE_PLUGINS = [
+    "funkwhale_api.contrib.scrobbler",
+]
+
+LOAD_CORE_PLUGINS = env.bool("FUNKWHALE_LOAD_CORE_PLUGINS", default=True)
+PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
+"""
+List of Funkwhale plugins to load.
+"""
+if LOAD_CORE_PLUGINS:
+    PLUGINS = CORE_PLUGINS + PLUGINS
+
+# Remove duplicates
+PLUGINS = list(OrderedDict.fromkeys(PLUGINS))
+
+if PLUGINS:
+    logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS))
+else:
+    logger.info("Running with no plugins")
+
+from .. import plugins  # noqa
 
+plugins.startup.autodiscover([p + ".funkwhale_startup" for p in PLUGINS])
+DEPENDENCIES = plugins.trigger_filter(plugins.PLUGINS_DEPENDENCIES, [], enabled=True)
+plugins.install_dependencies(DEPENDENCIES)
 FUNKWHALE_HOSTNAME = None
 FUNKWHALE_HOSTNAME_SUFFIX = env("FUNKWHALE_HOSTNAME_SUFFIX", default=None)
 FUNKWHALE_HOSTNAME_PREFIX = env("FUNKWHALE_HOSTNAME_PREFIX", default=None)
@@ -144,22 +174,7 @@ FUNKWHALE_SPA_REWRITE_MANIFEST_URL = env.bool(
 
 APP_NAME = "Funkwhale"
 
-# XXX: for backward compat with django 2.2, remove this when django 2.2 support is dropped
-os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = env.bool(
-    "DJANGO_ALLOW_ASYNC_UNSAFE", default="true"
-)
-
-# XXX: deprecated, see #186
-FEDERATION_ENABLED = env.bool("FEDERATION_ENABLED", default=True)
 FEDERATION_HOSTNAME = env("FEDERATION_HOSTNAME", default=FUNKWHALE_HOSTNAME).lower()
-# XXX: deprecated, see #186
-FEDERATION_COLLECTION_PAGE_SIZE = env.int("FEDERATION_COLLECTION_PAGE_SIZE", default=50)
-# XXX: deprecated, see #186
-FEDERATION_MUSIC_NEEDS_APPROVAL = env.bool(
-    "FEDERATION_MUSIC_NEEDS_APPROVAL", default=True
-)
-# XXX: deprecated, see #186
-FEDERATION_ACTOR_FETCH_DELAY = env.int("FEDERATION_ACTOR_FETCH_DELAY", default=60 * 12)
 FEDERATION_SERVICE_ACTOR_USERNAME = env(
     "FEDERATION_SERVICE_ACTOR_USERNAME", default="service"
 )
@@ -247,16 +262,6 @@ LOCAL_APPS = (
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
 
-
-PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p]
-"""
-List of Funkwhale plugins to load.
-"""
-if PLUGINS:
-    logger.info("Running with the following plugins enabled: %s", ", ".join(PLUGINS))
-else:
-    logger.info("Running with no plugins")
-
 ADDITIONAL_APPS = env.list("ADDITIONAL_APPS", default=[])
 """
 List of Django apps to load in addition to Funkwhale plugins and apps.
@@ -265,25 +270,32 @@ INSTALLED_APPS = (
     DJANGO_APPS
     + THIRD_PARTY_APPS
     + LOCAL_APPS
-    + tuple(["{}.apps.Plugin".format(p) for p in PLUGINS])
     + tuple(ADDITIONAL_APPS)
+    + tuple(plugins.trigger_filter(plugins.PLUGINS_APPS, [], enabled=True))
 )
 
 # MIDDLEWARE CONFIGURATION
 # ------------------------------------------------------------------------------
 ADDITIONAL_MIDDLEWARES_BEFORE = env.list("ADDITIONAL_MIDDLEWARES_BEFORE", default=[])
-MIDDLEWARE = tuple(ADDITIONAL_MIDDLEWARES_BEFORE) + (
-    "django.middleware.security.SecurityMiddleware",
-    "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "corsheaders.middleware.CorsMiddleware",
-    "funkwhale_api.common.middleware.SPAFallbackMiddleware",
-    "django.contrib.sessions.middleware.SessionMiddleware",
-    "django.middleware.common.CommonMiddleware",
-    "django.middleware.csrf.CsrfViewMiddleware",
-    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    "django.contrib.messages.middleware.MessageMiddleware",
-    "funkwhale_api.users.middleware.RecordActivityMiddleware",
-    "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
+MIDDLEWARE = (
+    tuple(plugins.trigger_filter(plugins.MIDDLEWARES_BEFORE, [], enabled=True))
+    + tuple(ADDITIONAL_MIDDLEWARES_BEFORE)
+    + (
+        "django.middleware.security.SecurityMiddleware",
+        "django.middleware.clickjacking.XFrameOptionsMiddleware",
+        "corsheaders.middleware.CorsMiddleware",
+        # needs to be before SPA middleware
+        "django.contrib.sessions.middleware.SessionMiddleware",
+        "django.middleware.common.CommonMiddleware",
+        "django.middleware.csrf.CsrfViewMiddleware",
+        # /end
+        "funkwhale_api.common.middleware.SPAFallbackMiddleware",
+        "django.contrib.auth.middleware.AuthenticationMiddleware",
+        "django.contrib.messages.middleware.MessageMiddleware",
+        "funkwhale_api.users.middleware.RecordActivityMiddleware",
+        "funkwhale_api.common.middleware.ThrottleStatusMiddleware",
+    )
+    + tuple(plugins.trigger_filter(plugins.MIDDLEWARES_AFTER, [], enabled=True))
 )
 
 # DEBUG
@@ -567,6 +579,8 @@ AUTHENTICATION_BACKENDS = (
     "funkwhale_api.users.auth_backends.AllAuthBackend",
 )
 SESSION_COOKIE_HTTPONLY = False
+SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", default=3600 * 25 * 60)
+
 # Some really nice defaults
 ACCOUNT_AUTHENTICATION_METHOD = "username_email"
 ACCOUNT_EMAIL_REQUIRED = True
@@ -861,6 +875,7 @@ REST_FRAMEWORK = {
     ),
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "funkwhale_api.common.authentication.OAuth2Authentication",
+        "funkwhale_api.common.authentication.ApplicationTokenAuthentication",
         "funkwhale_api.common.authentication.JSONWebTokenAuthenticationQS",
         "funkwhale_api.common.authentication.BearerTokenHeaderAuth",
         "funkwhale_api.common.authentication.JSONWebTokenAuthentication",
@@ -998,6 +1013,10 @@ THROTTLING_RATES = {
         "rate": THROTTLING_USER_RATES.get("oauth-revoke-token", "100/hour"),
         "description": "OAuth token deletion",
     },
+    "login": {
+        "rate": THROTTLING_USER_RATES.get("login", "30/hour"),
+        "description": "Login",
+    },
     "jwt-login": {
         "rate": THROTTLING_USER_RATES.get("jwt-login", "30/hour"),
         "description": "JWT token creation",
@@ -1106,11 +1125,6 @@ Exemples:
 CSRF_USE_SESSIONS = True
 SESSION_ENGINE = "django.contrib.sessions.backends.cache"
 
-# Playlist settings
-# XXX: deprecated, see #186
-PLAYLISTS_MAX_TRACKS = env.int("PLAYLISTS_MAX_TRACKS", default=250)
-
-
 ACCOUNT_USERNAME_BLACKLIST = [
     "funkwhale",
     "library",
@@ -1147,8 +1161,6 @@ EXTERNAL_REQUESTS_TIMEOUT = env.int("EXTERNAL_REQUESTS_TIMEOUT", default=10)
 """
 Default timeout for external requests.
 """
-# XXX: deprecated, see #186
-API_AUTHENTICATION_REQUIRED = env.bool("API_AUTHENTICATION_REQUIRED", True)
 
 MUSIC_DIRECTORY_PATH = env("MUSIC_DIRECTORY_PATH", default=None)
 """
@@ -1192,7 +1204,7 @@ On non-docker setup, you don't need to configure this setting.
 """
 # When this is set to default=True, we need to reenable migration music/0042
 # to ensure data is populated correctly on existing pods
-MUSIC_USE_DENORMALIZATION = env.bool("MUSIC_USE_DENORMALIZATION", default=False)
+MUSIC_USE_DENORMALIZATION = env.bool("MUSIC_USE_DENORMALIZATION", default=True)
 
 USERS_INVITATION_EXPIRATION_DAYS = env.int(
     "USERS_INVITATION_EXPIRATION_DAYS", default=14
@@ -1211,9 +1223,13 @@ VERSATILEIMAGEFIELD_RENDITION_KEY_SETS = {
     "attachment_square": [
         ("original", "url"),
         ("medium_square_crop", "crop__200x200"),
+        ("large_square_crop", "crop__600x600"),
     ],
 }
-VERSATILEIMAGEFIELD_SETTINGS = {"create_images_on_demand": False}
+VERSATILEIMAGEFIELD_SETTINGS = {
+    "create_images_on_demand": False,
+    "jpeg_resize_quality": env.int("THUMBNAIL_JPEG_RESIZE_QUALITY", default=95),
+}
 RSA_KEY_SIZE = 2048
 # for performance gain in tests, since we don't need to actually create the
 # thumbnails
@@ -1259,8 +1275,6 @@ FUNKWHALE_SUPPORT_MESSAGE_DELAY = env.int("FUNKWHALE_SUPPORT_MESSAGE_DELAY", def
 """
 Delay in days after signup before we show the "support Funkwhale" message
 """
-# XXX Stable release: remove
-USE_FULL_TEXT_SEARCH = env.bool("USE_FULL_TEXT_SEARCH", default=True)
 
 MIN_DELAY_BETWEEN_DOWNLOADS_COUNT = env.int(
     "MIN_DELAY_BETWEEN_DOWNLOADS_COUNT", default=60 * 60 * 6

api/config/urls.py

@@ -8,7 +8,9 @@ from django.conf.urls.static import static
 from funkwhale_api.common import admin
 from django.views import defaults as default_views
 
+from config import plugins
 
+plugins_patterns = plugins.trigger_filter(plugins.URLS, [], enabled=True)
 urlpatterns = [
     # Django Admin, use {% url 'admin:index' %}
     url(settings.ADMIN_URL, admin.site.urls),
@@ -21,8 +23,7 @@ urlpatterns = [
     ),
     url(r"^api/v1/auth/", include("funkwhale_api.users.rest_auth_urls")),
     url(r"^accounts/", include("allauth.urls")),
-    # Your stuff: custom urls includes go here
-]
+] + plugins_patterns
 
 if settings.DEBUG:
     # This allows the error pages to be debugged during development, just visit

api/funkwhale_api/__init__.py

 # -*- coding: utf-8 -*-
-__version__ = "0.21.2"
+__version__ = "1.0"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

api/funkwhale_api/audio/filters.py

@@ -41,7 +41,7 @@ class ChannelFilter(moderation_filters.HiddenContentFilterSet):
 
     class Meta:
         model = models.Channel
-        fields = ["q", "scope", "tag", "subscribed", "ordering", "external"]
+        fields = []
         hidden_content_fields_mapping = moderation_filters.USER_FILTER_CONFIG["CHANNEL"]
 
     def filter_subscribed(self, queryset, name, value):

api/funkwhale_api/audio/serializers.py

@@ -235,6 +235,7 @@ class ChannelUpdateSerializer(serializers.Serializer):
 class ChannelSerializer(serializers.ModelSerializer):
     artist = serializers.SerializerMethodField()
     actor = serializers.SerializerMethodField()
+    downloads_count = serializers.SerializerMethodField()
     attributed_to = federation_serializers.APIActorSerializer()
     rss_url = serializers.CharField(source="get_rss_url")
     url = serializers.SerializerMethodField()
@@ -250,6 +251,7 @@ class ChannelSerializer(serializers.ModelSerializer):
             "metadata",
             "rss_url",
             "url",
+            "downloads_count",
         ]
 
     def get_artist(self, obj):
@@ -264,6 +266,9 @@ class ChannelSerializer(serializers.ModelSerializer):
     def get_subscriptions_count(self, obj):
         return obj.actor.received_follows.exclude(approved=False).count()
 
+    def get_downloads_count(self, obj):
+        return getattr(obj, "_downloads_count", None)
+
     def get_actor(self, obj):
         if obj.attributed_to == actors.get_service_actor():
             return None
@@ -824,7 +829,12 @@ def rss_serialize_item(upload):
         "enclosure": [
             {
                 # we enforce MP3, since it's the only format supported everywhere
-                "url": federation_utils.full_url(upload.get_listen_url(to="mp3")),
+                "url": federation_utils.full_url(
+                    reverse(
+                        "api:v1:stream-detail", kwargs={"uuid": str(upload.track.uuid)}
+                    )
+                    + ".mp3"
+                ),
                 "length": upload.size or 0,
                 "type": "audio/mpeg",
             }

api/funkwhale_api/audio/views.py

@@ -7,7 +7,7 @@ from rest_framework import viewsets
 
 from django import http
 from django.db import transaction
-from django.db.models import Count, Prefetch, Q
+from django.db.models import Count, Prefetch, Q, Sum
 from django.utils import timezone
 
 from funkwhale_api.common import locales
@@ -93,6 +93,14 @@ class ChannelViewSet(
             return serializers.ChannelUpdateSerializer
         return serializers.ChannelCreateSerializer
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        if self.action == "retrieve":
+            queryset = queryset.annotate(
+                _downloads_count=Sum("artist__tracks__downloads_count")
+            )
+        return queryset
+
     def perform_create(self, serializer):
         return serializer.save(attributed_to=self.request.user.actor)
 

api/funkwhale_api/cli/main.py

@@ -3,6 +3,8 @@ import sys
 
 from . import base
 from . import library  # noqa
+from . import media  # noqa
+from . import plugins  # noqa
 from . import users  # noqa
 
 from rest_framework.exceptions import ValidationError

api/funkwhale_api/cli/media.py

+import click
+
+from django.core.cache import cache
+from django.conf import settings
+from django.core.files.storage import default_storage
+
+from versatileimagefield.image_warmer import VersatileImageFieldWarmer
+from versatileimagefield import settings as vif_settings
+
+from funkwhale_api.common import utils as common_utils
+from funkwhale_api.common.models import Attachment
+
+from . import base
+
+
+@base.cli.group()
+def media():
+    """Manage media files (avatars, covers, attachments…)"""
+    pass
+
+
+@media.command("generate-thumbnails")
+@click.option("-d", "--delete", is_flag=True)
+def generate_thumbnails(delete):
+    """
+    Generate thumbnails for all images (avatars, covers, etc.).
+
+    This can take a long time and generate a lot of I/O depending of the size
+    of your library.
+    """
+    click.echo("Deleting existing thumbnails…")
+    if delete:
+        try:
+            # FileSystemStorage doesn't support deleting a non-empty directory
+            # so we reimplemented a method to do so
+            default_storage.force_delete("__sized__")
+        except AttributeError:
+            # backends doesn't support directory deletion
+            pass
+    MODELS = [
+        (Attachment, "file", "attachment_square"),
+    ]
+    for model, attribute, key_set in MODELS:
+        click.echo(
+            "Generating thumbnails for {}.{}…".format(model._meta.label, attribute)
+        )
+        qs = model.objects.exclude(**{"{}__isnull".format(attribute): True})
+        qs = qs.exclude(**{attribute: ""})
+        cache_key = "*{}{}*".format(
+            settings.MEDIA_URL, vif_settings.VERSATILEIMAGEFIELD_SIZED_DIRNAME
+        )
+        entries = cache.keys(cache_key)
+        if entries:
+            click.echo(
+                "  Clearing {} cache entries: {}…".format(len(entries), cache_key)
+            )
+            for keys in common_utils.batch(iter(entries)):
+                cache.delete_many(keys)
+        warmer = VersatileImageFieldWarmer(
+            instance_or_queryset=qs,
+            rendition_key_set=key_set,
+            image_attr=attribute,
+            verbose=True,
+        )
+        click.echo("  Creating images")
+        num_created, failed_to_create = warmer.warm()
+        click.echo(
+            "  {} created, {} in error".format(num_created, len(failed_to_create))
+        )

api/funkwhale_api/cli/plugins.py

+import os
+import subprocess
+import sys
+
+import click
+from django.conf import settings
+
+
+from . import base
+
+
+@base.cli.group()
+def plugins():
+    """Manage plugins"""
+    pass
+
+
+@plugins.command("install")
+@click.argument("plugin", nargs=-1)
+def install(plugin):
+    """
+    Install a plugin from a given URL (zip, pip or git are supported)
+    """
+    if not plugin:
+        return click.echo("No plugin provided")
+
+    click.echo("Installing plugins…")
+    pip_install(list(plugin), settings.FUNKWHALE_PLUGINS_PATH)
+
+
+def pip_install(deps, target):
+    if not deps:
+        return
+    pip_path = os.path.join(os.path.dirname(sys.executable), "pip")
+    subprocess.check_call([pip_path, "install", "-t", target] + deps)

api/funkwhale_api/common/apps.py

 from django.apps import AppConfig, apps
+from django.conf import settings
+
+from config import plugins
 
 from . import mutations
 from . import utils
@@ -13,3 +16,6 @@ class CommonConfig(AppConfig):
         app_names = [app.name for app in apps.app_configs.values()]
         mutations.registry.autodiscover(app_names)
         utils.monkey_patch_request_build_absolute_uri()
+        plugins.startup.autodiscover([p + ".funkwhale_ready" for p in settings.PLUGINS])
+        for p in plugins._plugins.values():
+            p["settings"] = plugins.load_settings(p["name"], p["settings"])

api/funkwhale_api/common/auth.py

-from urllib.parse import parse_qs
-
-from django.contrib.auth.models import AnonymousUser
-from rest_framework import exceptions
-from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
-
-from funkwhale_api.users.models import User
-
-
-class TokenHeaderAuth(BaseJSONWebTokenAuthentication):
-    def get_jwt_value(self, request):
-
-        try:
-            qs = request.get("query_string", b"").decode("utf-8")
-            parsed = parse_qs(qs)
-            token = parsed["token"][0]
-        except KeyError:
-            raise exceptions.AuthenticationFailed("No token")
-
-        if not token:
-            raise exceptions.AuthenticationFailed("Empty token")
-
-        return token
-
-
-class TokenAuthMiddleware:
-    def __init__(self, inner):
-        # Store the ASGI application we were passed
-        self.inner = inner
-
-    def __call__(self, scope):
-        # XXX: 1.0 remove this, replace with websocket/scopedtoken
-        auth = TokenHeaderAuth()
-        try:
-            user, token = auth.authenticate(scope)
-        except (User.DoesNotExist, exceptions.AuthenticationFailed):
-            user = AnonymousUser()
-
-        scope["user"] = user
-        return self.inner(scope)

api/funkwhale_api/common/authentication.py

@@ -12,6 +12,8 @@ from rest_framework import exceptions
 from rest_framework_jwt import authentication
 from rest_framework_jwt.settings import api_settings
 
+from funkwhale_api.users import models as users_models
+
 
 def should_verify_email(user):
     if user.is_superuser:
@@ -46,6 +48,36 @@ class OAuth2Authentication(BaseOAuth2Authentication):
             resend_confirmation_email(request, e.user)
 
 
+class ApplicationTokenAuthentication(object):
+    def authenticate(self, request):
+        try:
+            header = request.headers["Authorization"]
+        except KeyError:
+            return
+
+        if "Bearer" not in header:
+            return
+
+        token = header.split()[-1].strip()
+
+        try:
+            application = users_models.Application.objects.exclude(user=None).get(
+                token=token
+            )
+        except users_models.Application.DoesNotExist:
+            return
+        user = users_models.User.objects.all().for_auth().get(id=application.user_id)
+        if not user.is_active:
+            msg = _("User account is disabled.")
+            raise exceptions.AuthenticationFailed(msg)
+
+        if should_verify_email(user):
+            raise UnverifiedEmail(user)
+
+        request.scopes = application.scope.split()
+        return user, None
+
+
 class BaseJsonWebTokenAuth(object):
     def authenticate(self, request):
         try:

api/funkwhale_api/common/dynamic_preferences_registry.py

 from dynamic_preferences import types
 from dynamic_preferences.registries import global_preferences_registry
 
-from funkwhale_api.common import preferences
-
 common = types.Section("common")
 
 
 @global_preferences_registry.register
-class APIAutenticationRequired(
-    preferences.DefaultFromSettingMixin, types.BooleanPreference
-):
+class APIAutenticationRequired(types.BooleanPreference):
     section = common
     name = "api_authentication_required"
     verbose_name = "API Requires authentication"
-    setting = "API_AUTHENTICATION_REQUIRED"
+    default = True
     help_text = (
         "If disabled, anonymous users will be able to query the API"
         "and access music data (as well as other data exposed in the API "