I am new to django, docker and scrapy and I am trying to run a django app that also uses scrapy (I basically create a django app that is also a scrapy app and try to call a spider from a django view). Despite specifying this scrapy
in requirements.txt and running pip from the Dockerfile, the dependencies are not installed in the container prior to running python manage.py runserver 0.0.0.0:8000
and the django app fails during the system checks, resulting in the web container exiting because of the following exception:
| Exception in thread django-main-thread:
web_1 | Traceback (most recent call last):
web_1 | File "/usr/local/lib/python3.7/threading.py", line 926, in _bootstrap_inner
web_1 | self.run()
web_1 | File "/usr/local/lib/python3.7/threading.py", line 870, in run
web_1 | self._target(*self._args, **self._kwargs)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/utils/autoreload.py", line 54, in wrapper
web_1 | fn(*args, **kwargs)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/core/management/commands/runserver.py", line 117, in inner_run
web_1 | self.check(display_num_errors=True)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/core/management/base.py", line 390, in check
web_1 | include_deployment_checks=include_deployment_checks,
web_1 | File "/usr/local/lib/python3.7/site-packages/django/core/management/base.py", line 377, in _run_checks
web_1 | return checks.run_checks(**kwargs)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/core/checks/registry.py", line 72, in run_checks
web_1 | new_errors = check(app_configs=app_configs)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/core/checks/urls.py", line 40, in check_url_namespaces_unique
web_1 | all_namespaces = _load_all_namespaces(resolver)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/core/checks/urls.py", line 57, in _load_all_namespaces
web_1 | url_patterns = getattr(resolver, 'url_patterns', [])
web_1 | File "/usr/local/lib/python3.7/site-packages/django/utils/functional.py", line 80, in __get__
web_1 | res = instance.__dict__[self.name] = self.func(instance)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/urls/resolvers.py", line 579, in url_patterns
web_1 | patterns = getattr(self.urlconf_module, "urlpatterns", self.urlconf_module)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/utils/functional.py", line 80, in __get__
web_1 | res = instance.__dict__[self.name] = self.func(instance)
web_1 | File "/usr/local/lib/python3.7/site-packages/django/urls/resolvers.py", line 572, in urlconf_module
web_1 | return import_module(self.urlconf_name)
web_1 | File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
web_1 | return _bootstrap._gcd_import(name[level:], package, level)
web_1 | File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
web_1 | File "<frozen importlib._bootstrap>", line 983, in _find_and_load
web_1 | File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
web_1 | File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
web_1 | File "<frozen importlib._bootstrap_external>", line 728, in exec_module
web_1 | File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
web_1 | File "/code/composeexample/urls.py", line 21, in <module>
web_1 | path('scrapy/', include('scrapy_app.urls')),
web_1 | File "/usr/local/lib/python3.7/site-packages/django/urls/conf.py", line 34, in include
web_1 | urlconf_module = import_module(urlconf_module)
web_1 | File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
web_1 | return _bootstrap._gcd_import(name[level:], package, level)
web_1 | File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
web_1 | File "<frozen importlib._bootstrap>", line 983, in _find_and_load
web_1 | File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
web_1 | File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
web_1 | File "<frozen importlib._bootstrap_external>", line 728, in exec_module
web_1 | File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
web_1 | File "/code/scrapy_app/urls.py", line 4, in <module>
web_1 | from scrapy_app import views
web_1 | File "/code/scrapy_app/views.py", line 1, in <module>
web_1 | from scrapy.crawler import CrawlerProcess
web_1 | ModuleNotFoundError: No module named 'scrapy'
I tried using pip3
instead of pip, pip install --no-cache-dir -r requirements.txt
, changing the order of the statements in the Dockerfile and I also checked that Scrapy==1.7.3
appears in requirements.txt
. Nothing seems to work.
This is my Dockerfile:
FROM python:3
ENV PYTHONUNBUFFERED 1
RUN mkdir /code
WORKDIR /code
COPY requirements.txt /code/
RUN pip install -r requirements.txt
COPY . /code/
And this is my docker-compose.yml:
version: '3'
services:
db:
image: postgres
web:
build: .
command: python manage.py runserver 0.0.0.0:8000
volumes:
- .:/code
ports:
- "8000:8000"
depends_on:
- db
requirements.txt