Commit 0a295cbd authored by Glenn Walbran's avatar Glenn Walbran Committed by Cecilia Vela Gurovic
Browse files

Bug 1895201 Docker images for Mahara

This commit adds docker images for building, testing and running Mahara.
The images are:
- mahara-base, has packages that are needed to both run and test Mahara.
  Base of an ubuntu base image.
- mahara-builder, image to be used for building and testing Mahara. Extends
  mahara-base and adds packages needed for building and testing
- mahara, image used for running Mahara. Has the Mahara code (that has
  already been built) baked into it.

Things to note:
- The configuration in the docker images is driven by environment variables.
  htdocs/config-environment.php is a config file variant that will look for
  config in from the environment variables else fall back to a default value.
  This file is copied to config.php as part of creating the mahara image.
- The mahara image only has support for http. If https is required (which is
  recommended) then this should be done by adding a reverse proxy whose job
  is to terminate the https.

Change-Id: I65af1231dc42684e596c6e236fcf1de166d1abc7
(cherry picked from commit 87e202ef)
parent c3d2fe39
#Start by ignoring everything
**
#Now we will allow only the paths that are needed
!Makefile
!docker
!htdocs
# Contains useful environment variable defaults for testing Mahara in the
# docker environments.
#
MAHARA_DOCKER_PORT=80
MAHARA_WWW_SERVERNAME=localdev.mahara.org
MAHARA_WWW_SERVEADMIN=webmaster@localhost
MAHARA_DB_TYPE
MAHARA_DB_HOST=mahara-db
MAHARA_DB_USER=mahara
MAHARA_DB_NAME=mahara
MAHARA_DB_PASSWD=mahara
MAHARA_DATA_ROOT
MAHARA_WWW_ROOT
MAHARA_SEND_EMAIL
MAHARA_SEND_ALL_EMAIL_TO
MAHARA_PRODUCTION_MODE
MAHARA_PERF_TO_FOOT
MAHARA_SKINS
MAHARA_ISOLOATED_INSTITUTIONS
MAHARA_DB_PREFIX
MAHARA_SITE_THEME_PREFS
MAHARA_CLEAN_URLS
MAHARA_PUBLIC_SEARCH_ALLOWED
MAHARA_PROBATION_ENABLED
MAHARA_SHOW_LOGIN_INSIDE_BLOCK
MAHARA_EXTERNAL_LOGIN
MAHARA_URL_SECRET=ZfzXfLQMd8TMu
MAHARA_PASSWORD_SALT_MAIN=RpKhcdNPdNt9T
MAHARA_PASSWORD_SALT_ALT1
MAHARA_SESSION_HANDLER=redis
MAHARA_REDIS_SERVER=mahara-redis
MAHARA_REDIS_SENTINEL_SERVERS
MAHARA_REDIS_MASTER_GROUP
MAHARA_REDIS_PREFIX
MAHARA_ELASTICSEARCH_HOST
MAHARA_ELASTICSEARCH_PORT
MAHARA_ELASTICSEARCH_SCHEME
MAHARA_ELASTICSEARCH_USERNAME
MAHARA_ELASTICSEARCH_PASSWD
MAHARA_ELASTICSEARCH_INDEXING_USERNAME
MAHARA_ELASTICSEARCH_INDEXING_PASSWD
MAHARA_ELASTICSEARCH_INDEX_NAME
MAHARA_ELASTICSEARCH_BYPASS_INDEX_NAME
MAHARA_ELASTICSEARCH_ANALYZER
MAHARA_ELASTICSEARCH_TYPES
MAHARA_ELASTICSEARCH_IGNORE_SSL
MAHARA_ELASTICSEARCH_REQUEST_LIMIT
MAHARA_ELASTICSEARCH_REDO_LIMIT
......@@ -2,6 +2,8 @@
.buildpath
.settings/
.settings
.envrc
behat-root/
logs
htdocs/config.php
htdocs/.project
......
# The mode that behat tests are run with.
BEHAT_MODE = rundebugheadless
# Can limit what behat tests are run, e.g `make -e BEHAT_TESTS=change_account_settings.feature behat`
BEHAT_TESTS = null
# ask for test reports to be generated possible values are <empty>, 'html', 'junit'
BEHAT_REPORT =
# The Ubuntu version that the Mahara base image will be based on
DOCKER_UBUNTU_VERSION = bionic
TEST_ADMIN_PASSWD = Kupuh1pa!
TEST_ADMIN_EMAIL = user@example.org
# Make expects targets to create a file that matches the target name
# unless the target is phony.
# Refer to: https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html
.PHONY: css clean-css help imageoptim installcomposer initcomposer cleanssphp ssphp \
cleanpdfexport pdfexport install phpunit behat minaccept jenkinsaccept securitycheck \
push security docker-image docker-builder
all: css
production = true
......@@ -24,12 +42,16 @@ help:
@echo "Run 'make' to do "build" Mahara (currently only CSS)"
@echo "Run 'make initcomposer' to install Composer and phpunit"
@echo "Run 'make phpunit' to execute phpunit tests"
@echo "Run 'make install' runs the Mahara install script"
@echo "Run 'make behat' to execute behat tests"
@echo "Run 'make ssphp' to install SimpleSAMLphp"
@echo "Run 'make cleanssphp' to remove SimpleSAMLphp"
@echo "Run 'make imageoptim' to losslessly optimise all images"
@echo "Run 'make minaccept' to run the quick pre-commit tests"
@echo "Run 'make checksignoff' to check that your commits are all Signed-off-by"
@echo "Run 'make push' to push your changes to the repo"
@echo "Run 'make docker-image' to build a Mahara docker image"
@echo "Run 'make docker-builder' builds the docker builder image required for docker-build"
imageoptim:
find . -iname '*.png' -exec optipng -o7 -q {} \;
......@@ -94,7 +116,10 @@ endif
vendorphpunit := $(shell external/vendor/bin/phpunit --version 2>/dev/null)
phpunit:
install:
php htdocs/admin/cli/install.php --adminpassword=$(TEST_ADMIN_PASSWD) --adminemail=$(TEST_ADMIN_EMAIL)
phpunit: install
@echo "Running phpunit tests..."
ifdef vendorphpunit
@external/vendor/bin/phpunit --log-junit logs/tests/phpunit-results.xml htdocs/
......@@ -102,6 +127,8 @@ else
@phpunit --log-junit logs/tests/phpunit-results.xml htdocs/
endif
behat:
./test/behat/mahara_behat.sh $(BEHAT_MODE) $(BEHAT_TESTS) $(BEHAT_REPORT)
revision := $(shell git rev-parse --verify HEAD 2>/dev/null)
whitelist := $(shell grep / test/WHITELIST | xargs -I entry find entry -type f | xargs -I file echo '! -path ' file 2>/dev/null)
......@@ -159,3 +186,25 @@ security: minaccept
git push gerrit HEAD:refs/drafts/20.10_STABLE/$(TAG); \
fi
ssh $(sshargs) gerrit set-reviewers --add \"Mahara Security Managers\" -- $(sha1chain)
# Builds Mahara server docker image
docker-image:
docker build --pull --file docker/Dockerfile.mahara-base \
--build-arg BASE_VERSION=$(DOCKER_UBUNTU_VERSION) \
--tag mahara-base:$(DOCKER_UBUNTU_VERSION) .
docker build --file docker/Dockerfile.mahara-web \
--build-arg BASE_IMAGE=mahara-base:$(DOCKER_UBUNTU_VERSION) \
--tag mahara .
# Builds a docker image that is able to build Mahara. Useful if you don't want
# to install dependencies on your system.
# The builder is made for the user that will use it. This is so that the built
# files are owned by the user and not some other user (eg not root)
docker-builder:
docker build --pull --file docker/Dockerfile.mahara-base \
--build-arg BASE_VERSION=$(DOCKER_UBUNTU_VERSION) \
--tag mahara-base:$(DOCKER_UBUNTU_VERSION) .
docker build --file docker/Dockerfile.mahara-builder \
--build-arg BASE_IMAGE=mahara-base:$(DOCKER_UBUNTU_VERSION) \
--build-arg IMAGE_UID=$(shell id -u) --build-arg IMAGE_GID=$(shell id -g) \
--tag mahara-builder .
# Base Mahara image containing packages that are needed to build/test and
# run a Mahara instance
ARG BASE_UBUNTU=docker.catalyst.net.nz/catalyst/ubuntu:bionic
FROM ${BASE_UBUNTU}
# enviroment variable as non interactive
ARG DEBIAN_FRONTEND=noninteractive
# update packages
# Chromium is used for pdf exports and for behat tests
RUN apt-get update && \
apt-get install -y \
ca-certificates \
chromium-browser \
php-cli \
php-curl \
php-dom \
php-gd \
php-json \
php-ldap \
php-mbstring \
php-pgsql \
php-redis \
php-xmlrpc \
php-zip \
php-xml
RUN mkdir -p /mahara/data && \
chmod 777 /mahara/data
VOLUME /mahara/data
ENTRYPOINT ["/bin/bash"]
# Creates an image that can be used to build and test Mahara.
#
# The image is created for the person that will use it to prevent built files
# being owned by other users.
#
ARG BASE_IMAGE=mahara-base
FROM ${BASE_IMAGE}
ARG IMAGE_UID=1000
ARG IMAGE_GID=1000
# Set the enviroment variable as non interactive
ARG DEBIAN_FRONTEND=noninteractive
# Setup the builder user
RUN set -o errexit -o nounset \
&& groupadd --system --gid ${IMAGE_GID} builder \
&& useradd --system --gid builder --uid ${IMAGE_UID} --shell /bin/bash --create-home builder
# Install dependencies
RUN apt-get update && apt-get -y install \
curl \
git \
lsof \
make \
nodejs-dev \
node-gyp \
npm \
openjdk-8-jre-headless \
unzip \
wget \
xvfb \
&& npm config set prefix /usr/local \
&& npm install -g gulp --silent
# Create docbuilder/source volume
# We expect this to be volume mounted to supply the document source files
VOLUME "/opt/mahara"
WORKDIR /opt/mahara
RUN chown ${IMAGE_UID}:${IMAGE_GID} /opt/mahara
USER builder
ENTRYPOINT ["make"]
# base image
ARG BASE_IMAGE=mahara-base
FROM ${BASE_IMAGE}
# enviroment variable as non interactive
ARG DEBIAN_FRONTEND=noninteractive
# update packages
RUN apt-get update && \
apt-get install -y \
apache2 \
libapache2-mod-php \
poppler-utils
RUN sed -i "s/^upload_max_filesize = .*/upload_max_filesize = 99M/; s/^post_max_size = .*/post_max_size = 100M/" /etc/php/*/cli/php.ini \
&& sed -i "s/^upload_max_filesize = .*/upload_max_filesize = 99M/; s/^post_max_size = .*/post_max_size = 100M/" /etc/php/*/apache2/php.ini
# Make the apache access.log and error.log files sym links to stdout/stderr.
# This makes all the logging appear in `docker container logs`
RUN ln -sf /dev/stdout /var/log/apache2/access.log && \
ln -sf /dev/stderr /var/log/apache2/error.log
COPY htdocs /mahara/htdocs
COPY docker/web/etc/apache2/ /etc/apache2/
# - sym link the config.php from config-environment.ph
# - disable the apache2 service because the container will run that process in the foreground.
# - disable the default site and enable the Mahara site.
RUN cd /mahara/htdocs && ln -sf config-environment.php config.php && \
update-rc.d apache2 disable && a2dissite 000-default && a2ensite mahara-http
Expose 80
# Run apache to bring up Mahara
CMD ["/usr/sbin/apache2ctl", "-DFOREGROUND"]
# Mahara Docker
This directory contains tools for using or testing Mahara using Docker.
Docker is a great way to try out Mahara without having the hassle of installing
dependencies yourself.
The instructions have been tested on Ubuntu.
## Docker quick start
Test if you have Docker:
```
docker run ubuntu echo "Yes, I have Docker."
```
If you have Docker, and it is set up, the last line of the output will have
"Yes, I have Docker.". You can skip the rest of this section.
Note for the installation steps below: If you work for a company that works
with Docker, you may wish to check your internal documentation first, as there
may be some special instructions for you to follow instead.
To install and configure Docker on a recent Ubuntu desktop:
```
sudo apt install docker.io
# Add yourself to the Docker group
sudo usermod -aG docker $USER
newgrp docker
```
Rerun the Docker test above to confirm.
Note: You shouldn't need to run Docker with sudo. Attempting to run some of the
commands in this README with sudo are likely to cause errors.
## Customising the Docker environment
Note that the following scripts and Docker commands can read a lot of config
variables from the environment.
If there are settings that you always want to make, having them set as
environment variables may make your life easier. For example:
```
export MAHARA_DOCKER_PORT=8081
```
Will make the `mahara` Docker container publish its http port to http://localhost:8081
Using a tool such as [Direnv](https://direnv.net/) can make this even easier
because you can define the variables persistently.
For example, for both local development and running phpunit or Behat tests, you
can use an environment variables file to house all necessary config settings
instead of putting them into config.php itself. You may not wish to do so for a
production environment though when you have multiple Mahara instances running
on a single server. To link the config.php file to the environment variables
file, run:
```
cd htdocs
ln -s config-environment.php config.php
cd ..
```
Note: Instead of putting values into config-environment.php, you can set up a .envrc
file to contain your custom variables as that will not be pushed to the repository
when you commit changes. It is excluded in .gitignore.
## Building and testing Mahara with Docker
Mahara has a mahara-builder Docker image that can be used to build and test
Mahara. This image can be built and updated with:
```
# This command should be run periodically (every week or two) to freshen up
# the image with any security patches from its base image
make docker-builder
```
This image contains the dependencies required for building and testing
Mahara. To use this image to execute `make` targets run:
```
./docker/make.sh <target>
e.g
./docker/make.sh css
./docker/make.sh phpunit
./docker/make.sh behat
# To run only some test features
./docker/make.sh -e BEHAT_TESTS=change_account_settings.feature behat
# To run some tests with browser head
./docker/make.sh -e BEHAT_MODE=rundebug -e BEHAT_TESTS=change_account_settings.feature behat
```
The running of phpunit and Behat tests via Docker is in the beginning stages,
and there are changes that need to be made to get it to run fully. We keep the
instructions in here though.
These targets also need a DB. You can start and stop a test Docker database
with these commands:
```
docker/test-db.sh start
# Run you tests while the DB is running
docker/test-db.sh stop
```
To access the database with psql run:
```
docker/test-db.sh psql
# To access the DB used in the docker-compose setup later on you need
# to change the docker network being used.
docker/test-db.sh -n docker_default psql
```
## Building the Mahara Docker image
The main Mahara Makefile has a `docker-image` target that is used to build the
Mahara Docker image. This will create a Docker image with Apache, PHP, and other
dependencies and Mahara.
```
make docker-image
```
This build target creates the Docker image from the contents of the ./htdocs
directory. Other build targets like `css` should have already been run to
create those assets.
## Using the Mahara Docker image
The Mahara Docker image could be used to create a real Mahara instance. The
`docker-compose.yaml` file could be customised for this or to see what
environment variables are required.
Note: A separate DB server (container) is also required.
Before you use your Mahara Docker image on a production instance, please test it thoroughly.
## Developer testing
When doing Mahara development it is often useful to run a Mahara server
which is serving the files from your local checkout. We can do this with
Docker by:
```
# First ensure you have a Mahara image
make docker-image
# You will need a database and most likely you will want to keep
# the same database for ongoing testing. In that case first create
# the volume to store the DB.
docker volume create mahara-dev-data
# Now start the DB
docker/test-db.sh -d mahara-dev-data start
# Now start Mahara
docker/test-mahara.sh start
# Stop Mahara and the DB
docker/test-mahara.sh stop
docker/test-db.sh stop
```
## Test environment
Docker compose is used to control the Mahara Docker environment. A database is required besides a
web server. We use PostgreSQL. Composer starts and stops both of these for us using the
following commands.
```
# Bootstrap the Docker dependencies first. This only needs to be
# done once.
docker network create mahara
docker volume create mahara-db
docker volume create mahara-data
docker volume create mahara-elastic
# Now start the Mahara environment
docker-compose -f docker/docker-compose.yaml up &
# Stop it once you are finished
docker-compose -f docker/docker-compose.yaml down
```
### External data volumes
This Docker compose setup uses external data volumes to persist files. This
is used to save the DB and the Mahara data directory. To set up the required
data volumes run the following commands:
```
docker volume create mahara-db
docker volume create mahara-data
```
But this could cause problems if you were to run a Mahara version that is older
than the last one that you have run. For example, these problems would be caused by the DB
appearing to be in an unknown future state from the point of view of the older
Mahara release.
To handle these situations without having to start from scratch every time,
the actual Docker volumes used can be changed using environment variables.
For example, the following commands could be used to run Mahara with a fresh
DB and data directory:
```
docker volume create mahara-db-fresh
docker volume create mahara-data-fresh
export MAHARA_DB_VOLUME=mahara-db-fresh
export MAHARA_DATA_VOLUME=mahara-data-fresh
docker-compose -f docker/docker-compose.yaml up &
```
{
"defaultAction": "SCMP_ACT_ERRNO",
"archMap": [
{
"architecture": "SCMP_ARCH_X86_64",
"subArchitectures": [
"SCMP_ARCH_X86",
"SCMP_ARCH_X32"
]
},
{
"architecture": "SCMP_ARCH_AARCH64",
"subArchitectures": [
"SCMP_ARCH_ARM"
]
},
{
"architecture": "SCMP_ARCH_MIPS64",
"subArchitectures": [
"SCMP_ARCH_MIPS",
"SCMP_ARCH_MIPS64N32"
]
},
{
"architecture": "SCMP_ARCH_MIPS64N32",
"subArchitectures": [
"SCMP_ARCH_MIPS",
"SCMP_ARCH_MIPS64"
]
},
{
"architecture": "SCMP_ARCH_MIPSEL64",
"subArchitectures": [
"SCMP_ARCH_MIPSEL",
"SCMP_ARCH_MIPSEL64N32"
]
},
{
"architecture": "SCMP_ARCH_MIPSEL64N32",
"subArchitectures": [
"SCMP_ARCH_MIPSEL",
"SCMP_ARCH_MIPSEL64"
]
},
{
"architecture": "SCMP_ARCH_S390X",
"subArchitectures": [
"SCMP_ARCH_S390"
]
}
],
"syscalls": [
{
"names": [
"accept",
"accept4",
"access",
"adjtimex",
"alarm",
"arch_prctl",
"bind",
"brk",
"capget",
"capset",
"chdir",
"chmod",
"chown",
"chown32",
"chroot",
"clock_getres",
"clock_gettime",
"clock_nanosleep",
"clone",
"close",
"connect",
"copy_file_range",
"creat",
"dup",
"dup2",
"dup3",
"epoll_create",
"epoll_create1",
"epoll_ctl",
"epoll_ctl_old",
"epoll_pwait",
"epoll_wait",
"epoll_wait_old",
"eventfd",
"eventfd2",
"execve",
"execveat",
"exit",
"exit_group",
"faccessat",
"fadvise64",
"fadvise64_64",
"fallocate",
"fanotify_init",
"fanotify_mark",
"fchdir",
"fchmod",
"fchmodat",
"fchown",
"fchown32",
"fchownat",
"fcntl",
"fcntl64",
"fdatasync",
"fgetxattr",
"flistxattr",
"flock",
"fork",
"fremovexattr",
"fsetxattr",
"fstat",
"fstat64",
"fstatat64",
"fstatfs",
"fstatfs64",
"fsync",
"ftruncate",
"ftruncate64",
"futex",
"futimesat",
"getcpu",
"getcwd",
"getdents",
"getdents64",
"getegid",
"getegid32",
"geteuid",
"geteuid32",
"getgid",
"getgid32",
"getgroups",
"getgroups32",
"getitimer",
"getpeername",
"getpgid",
"getpgrp",
"getpid",
"getppid",
"getpriority",
"getrandom",
"getresgid",
"getresgid32",
"getresuid",
"getresuid32",
"getrlimit",
"get_robust_list",
"getrusage",
"getsid",