Nginx as CDN POP

This released on ubuntu with packages nginx and nginx-extras (some modules used additionally that not avail on Alma packages, that’s why ubuntu).

We will store cached files in /cdncache and logs in /var/log/nginx/cdnlogs.

Our CDN URL is yourcdn1.domain.name for site1.dev and yourcdn2.domain.name for site2.dev

We will server static files (mp4,jpg,png,gif,jpeg,js,ico,html,htm,webp,css,mp3,wav,swf,mov,doc,pdf,xls,ppt,docx,pptx,xlsx,ttf,woff,woff2), accept only GET queries and disallow listing for /.

user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;

worker_rlimit_nofile 100000;
pcre_jit on;
events {
  use epoll;
  worker_connections 16000;
  multi_accept on;
}
http {
# IP whitelist to which no conn/rate restrictions should be applied
  geo $ip_whitelist {
    default        0;
    127.0.0.1      1;
    10.225.1.0/24  1;
    YOUR.WEBSITE.IP/32  1;
  }
  map_hash_bucket_size 256;
  map $ip_whitelist $limited_ip {
    0  $binary_remote_addr;
    1  "";
  }
limit_conn_zone $limited_ip zone=connsPerIP:20m;
  limit_conn connsPerIP 100;
  limit_conn_status 429;
limit_req_zone $limited_ip zone=reqsPerMinutePerIP:50m rate=500r/m;
  limit_req zone=reqsPerMinutePerIP burst=700 nodelay;
  limit_req_status 429;
client_max_body_size 64k;
  client_header_timeout 10s;
  client_body_timeout 10s;
  client_body_buffer_size 16k;
  client_header_buffer_size 4k;
send_timeout 10s;
  connection_pool_size 512;
  large_client_header_buffers 8 16k;
  request_pool_size 4k;
http2_idle_timeout 60s;
  http2_recv_timeout 10s;
  http2_chunk_size 16k;
server_tokens off;
  more_set_headers "Server: My-CDN";
include /etc/nginx/mime.types;
  variables_hash_bucket_size 128;

gzip on;
  gzip_static on; # searches for the *.gz file and returns it directly from disk (compression is provided by our extra process in the background)
  gzip_disable "msie6";
  gzip_min_length 4096;
  gzip_buffers 16 64k;
  gzip_vary on;
  gzip_proxied any;
  gzip_types image/svg+xml text/plain text/css application/json application/x-javascript application/javascript text/xml application/xml application/xml+rss text/javascript text/x-component font/truetype font/opentype image/x-icon;
  gzip_comp_level 4;
# If you compile nginx with brotli support uncomment this strings
#brotli on;
#  brotli_static on; # searches for the *.br file and returns it directly from the disk (compression is provided by our extra process in the background)
#  brotli_types text/plain text/css application/javascript application/json image/svg+xml application/xml+rss;
#  brotli_comp_level 6;
output_buffers 1 32k;
  postpone_output 1460;
sendfile on;
  sendfile_max_chunk 1m;
  tcp_nopush on;
  tcp_nodelay on;
keepalive_timeout 10 10;
  ignore_invalid_headers on;
  reset_timedout_connection on;
open_file_cache          max=50000 inactive=30s;
  open_file_cache_valid    10s;
  open_file_cache_min_uses 2;
  open_file_cache_errors   on;
proxy_buffering           on;
  proxy_buffer_size         16k;
  proxy_buffers             64 16k;
  proxy_temp_path           /cdncache;
  proxy_cache_min_uses      2;
proxy_ignore_client_abort on;
  proxy_intercept_errors    on;
  proxy_next_upstream       error timeout invalid_header http_500 http_502 http_503 http_504;
  proxy_redirect            off;
  proxy_connect_timeout     60;
  proxy_send_timeout        180;
  proxy_cache_lock          on;
  proxy_read_timeout        10s;
# setting up trusted IP subnets to respect X-Forwarded-For header (for multi-level proxy setup)
  set_real_ip_from          127.0.0.1/32;
  set_real_ip_from          10.1.2.0/24;
  real_ip_header            X-Forwarded-For;
  real_ip_recursive         on;

include /etc/nginx/conf.d/upstreams.conf;
include /etc/nginx/sites-enabled/*;

}

Next we will specify our upstreams (site, that we will cache) in /etc/nginx/conf.d/upstreams.conf

We will limit max cache size to 4GB. You should create folders /cdncache/site1.dev and /cdncache/site2.dev manually.

upstream site1_cdn {
    server site1.dev:443 max_conns=50;
keepalive 50;
    keepalive_requests 50;
    keepalive_timeout 5s;
  }
proxy_cache_path /cdncache/site1.dev levels=1:2 keys_zone=cdn_cache_site1:10m max_size=4g inactive=60m use_temp_path=off;

upstream site2_cdn {
    server site2.dev:443 max_conns=50;
keepalive 50;
    keepalive_requests 50;
    keepalive_timeout 5s;
  }
proxy_cache_path /cdncache/site2.dev levels=1:2 keys_zone=cdn_cache_site2:10m max_size=4g inactive=60m use_temp_path=off;

/etc/nginx/sites-enables/site1.dev.conf

server {
#  listen 443 ssl;
listen YOUR.CDN.POP.IPV4:443 ssl default_server http2 reuseport deferred backlog=32768;
listen [YOUR.CDN.POP.IPV6]:443 ssl default_server http2 reuseport deferred backlog=32768;
  server_name yourcdn1.domain.name;

  ssl_certificate /etc/letsencrypt/live/domain.name/fullchain.pem;
  ssl_certificate_key /etc/letsencrypt/live/domain.name/privkey.pem;

  error_log /var/log/nginx/cdnlogs/site1.dev/error_log;
#  access_log /var/log/nginx/cdnlogs/site1.dev/access_log;
  access_log off;
# if you use IPv6 just remove ipv6=off
  resolver 1.1.1.1 ipv6=off valid=30s;

location ~* .(mp4|jpg|png|gif|jpeg|js|ico|html|htm|webp|css|mp3|wav|swf|mov|doc|pdf|xls|ppt|docx|pptx|xlsx|ttf|woff|woff2)$ {
#  location / {
    proxy_cache cdn_cache_site1;
    proxy_cache_key $uri$is_args$args;
    proxy_cache_valid 90d;
    proxy_pass https://site1.dev;
#    add_header Access-Control-Allow-Origin https://site1.dev;
      set $headerCorsAllowOrigin "";
      if ($http_origin ~ '^https?://(localhost|yourcdn1\.domain\.name|site1\.dev)') {
#          set $headerCorsAllowOrigin "$http_origin";
      add_header Access-Control-Allow-Origin "$http_origin";
      }
      if ($request_method = 'OPTIONS') {
          more_set_headers "Access-Control-Allow-Origin: $headerCorsAllowOrigin";
          more_set_headers "Access-Control-Allow-Methods: GET, HEAD, OPTIONS";
          more_set_headers "Access-Control-Max-Age: 3600";
          more_set_headers "Content-Length: 0";
          return 204;
      }
      # we allow to load content only from the original domain (e.g. it prevents displaying our images on foreign domains)
      valid_referers none blocked server_names site1.dev;
      if ($invalid_referer) {
          more_set_headers "Content-Type: application/json";
          return 403 '{"code": 403, "message": "Forbidden Resource - invalid referer"}';
      }

      set $webp "";
      set $file_for_webp "";
      if ($http_accept ~* webp) {
          set $webp "A";
      }
      if ($request_filename ~ (.+\.(png|jpe?g))$) {
          set $file_for_webp $1;
      }
      if (-f $file_for_webp.webp) {
          set $webp "${webp}E";
      }
      if ($webp = AE) {
          rewrite ^/(.+)$ /webp/$1 last;
      }
    limit_except GET {
        deny  all;
    }

    }
location / {
        return 403;
}
}

The same will do for site2.dev /etc/nginx/sites-enabled/site2.dev.conf

server {
#  listen 443 ssl;
listen YOUR.CDN.POP.IPV4:443;
listen [YOUR.CDN.POP.IPV6]:443;
  server_name yourcdn2.domain.name;

  ssl_certificate /etc/letsencrypt/live/domain.name/fullchain.pem;
  ssl_certificate_key /etc/letsencrypt/live/domain.name/privkey.pem;

  error_log /var/log/nginx/cdnlogs/domain.name/error_log;
#  access_log /var/log/nginx/cdnlogs/domain.name/access_log;
  access_log off;
# For enabling ipv6 upstream queries disable ipv6=off
  resolver 1.1.1.1 ipv6=off valid=30s;

location ~* .(mp4|jpg|png|gif|jpeg|js|ico|html|htm|webp|css|mp3|wav|swf|mov|doc|pdf|xls|ppt|docx|pptx|xlsx|ttf|woff|woff2)$ {
#  location / {
    proxy_cache cdn_cache_site2;
    proxy_cache_key $uri$is_args$args;
    proxy_cache_valid 90d;
    proxy_pass https://site2.dev;
#    add_header Access-Control-Allow-Origin https://site2.dev;
      set $headerCorsAllowOrigin "";
      if ($http_origin ~ '^https?://(localhost|yourcdn2\.domain\.name|site2\.dev)') {
#          set $headerCorsAllowOrigin "$http_origin";
      add_header Access-Control-Allow-Origin "$http_origin";
      }
      if ($request_method = 'OPTIONS') {
          more_set_headers "Access-Control-Allow-Origin: $headerCorsAllowOrigin";
          more_set_headers "Access-Control-Allow-Methods: GET, HEAD, OPTIONS";
          more_set_headers "Access-Control-Max-Age: 3600";
          more_set_headers "Content-Length: 0";
          return 204;
      }
      # we allow to load content only from the original domain (e.g. it prevents displaying our images on foreign domains)
      valid_referers none blocked server_names site2.dev;
      if ($invalid_referer) {
          more_set_headers "Content-Type: application/json";
          return 403 '{"code": 403, "message": "Forbidden Resource - invalid referer"}';
      }

      set $webp "";
      set $file_for_webp "";
      if ($http_accept ~* webp) {
          set $webp "A";
      }
      if ($request_filename ~ (.+\.(png|jpe?g))$) {
          set $file_for_webp $1;
      }
      if (-f $file_for_webp.webp) {
          set $webp "${webp}E";
      }
      if ($webp = AE) {
          rewrite ^/(.+)$ /webp/$1 last;
      }
    limit_except GET {
        deny  all;
    }

    }
location / {
        return 403;
}
}

That’s all. Check configuration with nginx -t and restart nginx with systemctl restart nginx

Leave a Reply

Your email address will not be published. Required fields are marked *