enabling to monitor outputs from tensorflow serving from server side

docker-compose.yml

 version: '3.2'
 services:
- prometheus:
-    image: prom/prometheus:latest
-    container_name: prometheus
+
+ 
+  flask:
+    build: ./flask
+    container_name: flask
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.5
+    expose:
+      - 8080
+    volumes:
+      - ./flask/app/logs:/app/app/logs
+
+
+  nginx:
+    build: ./nginx
+    container_name: nginx
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.6
     ports:
+      - "80:80"
+
+
+  prometheus: 
+    image: prom/prometheus:latest 
+    container_name: prometheus 
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.7
+    ports: 
     - 9090:9090
-    command:
-    - --config.file=/etc/prometheus/prometheus.yml
-    volumes:
-    - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+    command: 
+    - --config.file=/etc/prometheus/prometheus.yml 
+    volumes: 
+    - ./prometheus.yml:/etc/prometheus/prometheus.yml 
+    
+    
+  grafana:
+    image: grafana/grafana:latest
+    restart: unless-stopped
+    user: "472"
+    container_name: grafana
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.8
     depends_on:
-    - tfx 
-
- tfx:
-  image: tensorflow/serving:latest
-  container_name: tfx 
-  ports:
-  - 8501:8501 
-  volumes:
-    - ./configs:/home/configs
-    - ./models/half_plus_two:/models/half_plus_two
-  environment:
-    - MODEL_NAME=half_plus_two
-  command:
-    - --monitoring_config_file=/home/configs/monitoring_config.txt
-
- grafana:
-   image: grafana/grafana:latest
-   container_name: grafana
-   ports:
-   - 3000:3000
-   depends_on:
-   - prometheus 
+      - prometheus
+    ports:
+      - 3000:3000
+    
+  loki:
+    image: grafana/loki:1.6.0
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.9
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
+
+  promtail:
+    image: grafana/promtail:1.6.0
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.10
+    volumes:
+      - ./flask/app/logs:/var/log
+    command: -config.file=/etc/promtail/config.yml
+
+  tfx:
+    image: tensorflow/serving:latest
+    networks: 
+      vpcbr:
+        ipv4_address: 10.5.0.12
+    container_name: tfx 
+    ports:
+    - 8501:8501 
+    volumes:
+      - ./configs:/home/configs
+      - ./models/half_plus_two:/models/half_plus_two
+    environment:
+      - MODEL_NAME=half_plus_two
+    command:
+      - --monitoring_config_file=/home/configs/monitoring_config.txt
+
+
+networks:
+  vpcbr:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.5.0.0/16
 
 

flask/.dockerignore

+env/
+__pycache__/
+.DS_Store
\ No newline at end of file

flask/Dockerfile

+# Use the Python3.7.2 image
+FROM python:3.7.2-stretch
+
+RUN pip install uwsgi
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the current directory contents into the container at /app 
+ADD . /app
+
+# Install the dependencies
+RUN pip install -r requirements.txt
+
+# RUN rm -rf multiproc-tmp
+# RUN mkdir multiproc-tmp
+# RUN export prometheus_multiproc_dir=multiproc-tmp
+# RUN gunicorn -c app/gunicorn_conf.py -w 4 app:app
+
+# run the command to start uWSGI
+CMD ["uwsgi", "app.ini"]
\ No newline at end of file

flask/app.ini

+[uwsgi]
+wsgi-file = run.py
+callable = app
+socket = :8080
+processes = 4
+threads = 2
+master = true
+chmod-socket = 660
+vacuum = true
+die-on-term = true
\ No newline at end of file

flask/app/__init__.py

+from flask import Flask
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from prometheus_client import make_wsgi_app
+
+app = Flask(__name__)
+
+from app import routes, monitoring
\ No newline at end of file

flask/app/loggers.py

+import logging
+"""
+Firts, create custom logs 
+
+"""
+logger = logging.getLogger('house_sales_prediction')
+logging.basicConfig(level=logging.INFO)
+
+#file handler 
+f_handler = logging.FileHandler('app/logs/app.log')
+#stream handler 
+s_handler = logging.StreamHandler()
+
+#create formatters 
+f_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
+s_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+#add formatd to handlers 
+f_handler.setFormatter(f_format)
+s_handler.setFormatter(s_format)
+
+#add handlers to logger object 
+
+logger.addHandler(f_handler)
+logger.addHandler(s_handler)
\ No newline at end of file

flask/app/logs/app.log

+house_sales_prediction - INFO - prediction 331199.18991777283
+house_sales_prediction - INFO - prediction 331199.18991777283
+house_sales_prediction - INFO - prediction 331199.18991777283
+house_sales_prediction - INFO - prediction 331199.18991777283
+house_sales_prediction - INFO - prediction 331199.18991777283
+house_sales_prediction - INFO - prediction 452498.1529080808
+house_sales_prediction - INFO - prediction 452498.1529080808
+house_sales_prediction - INFO - prediction 452498.1529080808
+house_sales_prediction - INFO - prediction 452498.1529080808
+house_sales_prediction - INFO - prediction 452498.1529080808
+house_sales_prediction - INFO - prediction 452498.1529080808
+house_sales_prediction - INFO - prediction 400000
+house_sales_prediction - INFO - prediction 400000
+house_sales_prediction - INFO - prediction 400000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 300000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 500000
+house_sales_prediction - INFO - prediction 500000
+house_sales_prediction - INFO - prediction 500000
+house_sales_prediction - INFO - prediction 500000
+house_sales_prediction - INFO - prediction 500000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 500000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 390000
+house_sales_prediction - INFO - prediction 0.5425775701605796
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - prediction 0.17854209927058173
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - prediction 0.13846843879212156
+house_sales_prediction - INFO - prediction 0.24119168591592122
+house_sales_prediction - INFO - prediction 0.9887184195457822
+house_sales_prediction - INFO - prediction 0.1820794019784513
+house_sales_prediction - INFO - prediction 0.94230237772786
+house_sales_prediction - INFO - prediction 0.3966121108597508
+house_sales_prediction - INFO - prediction 0.8649328206648436
+house_sales_prediction - INFO - prediction 0.048796212727250654
+house_sales_prediction - INFO - prediction 0.250700669672711
+house_sales_prediction - INFO - prediction 0.864823545245786
+house_sales_prediction - INFO - prediction 0.4483214742352195
+house_sales_prediction - INFO - prediction 0.7334921322550184
+house_sales_prediction - INFO - prediction 0.9446087650176831
+house_sales_prediction - INFO - prediction 0.21429991709962026
+house_sales_prediction - INFO - prediction 0.8491365927269025
+house_sales_prediction - INFO - prediction 0.09458485903774183
+house_sales_prediction - INFO - prediction 0.9655411620545572
+house_sales_prediction - INFO - prediction 0.6997399209665406
+house_sales_prediction - INFO - prediction 0.9968315788488228
+house_sales_prediction - INFO - prediction 0.684701283928769
+house_sales_prediction - INFO - prediction 0.30945395585545177
+house_sales_prediction - INFO - prediction 0.8431558600561972
+house_sales_prediction - INFO - prediction 0.21825436033012424
+house_sales_prediction - INFO - prediction 0.7470552878849573
+house_sales_prediction - INFO - prediction 0.8880723748956488
+house_sales_prediction - INFO - prediction 0.8355532742610174
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 480000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 330000
+house_sales_prediction - INFO - prediction 590000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 570000
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 360000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 580000
+house_sales_prediction - INFO - prediction 370000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 350000
+house_sales_prediction - INFO - prediction 490000
+house_sales_prediction - INFO - prediction 490000
+house_sales_prediction - INFO - prediction 490000
+house_sales_prediction - INFO - prediction 470000
+house_sales_prediction - INFO - prediction 470000
+house_sales_prediction - INFO - prediction 470000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - prediction 320000
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - prediction 380000.0
+house_sales_prediction - INFO - prediction 370000.0
+house_sales_prediction - INFO - prediction 360000.0
+house_sales_prediction - INFO - prediction 340000.0
+house_sales_prediction - INFO - server is up
+house_sales_prediction - INFO - server is up

flask/app/monitoring.py

+from prometheus_client import start_http_server, Gauge, CollectorRegistry
+import random
+
+
+
+#To avoid metrics fluctuations 
+# Multiprocessing setup
+# Cf. https://github.com/prometheus/client_python#multiprocess-mode-gunicorn
+# registry = CollectorRegistry()
+# multiprocess.MultiProcessCollector(registry)
+
+#monitoring predictions 
+prediction_gauge = Gauge('predictions', 'Discritpion of prediction in Gauge',['order'])
+
+
+
+# COUNTER = Counter('invoked_hello_world','How many times hello world is invoked')
+# PREDICTION_COUNTER = Counter('invoked_predictions','How many times prediction request is sent')
+# INFO = Info('prediction_value','Meaningless random prediction info')
+# SUMMARY = Summary('request_latency_seconds', 'Description of summary')
+# HISTOGRAM = Histogram('prediction', 'Description of prediction in Histogram')
+
+

flask/app/routes.py

+from app import app
+from app.monitoring import *
+from flask import Flask,request,Response,jsonify
+from prometheus_client import make_wsgi_app
+from prometheus_client import start_http_server, Summary, Gauge, Counter, multiprocess
+from prometheus_client import multiprocess, CollectorRegistry
+import prometheus_client
+import json 
+# import uwsgi
+import requests
+
+from app.train_and_predict import *
+
+import os 
+
+
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from prometheus_client import make_wsgi_app
+
+from app.loggers import *
+
+
+
+
+
+"""
+define app routes
+
+"""
+
+# prometheus_client.values.ValueClass = prometheus_client.values.MultiProcessValue(process_identifier=uwsgi.worker_id)
+
+
+
+@app.route('/')
+def hello_world():
+   logger.info('server is up')
+   return "Server is running successfully."
+
+@app.route('/prediction', methods=['POST'])
+def predict_from_tf_serving():
+   to_predict = request.json
+   result = requests.post("http://10.5.0.12:8501/v1/models/half_plus_two:predict",data=json.dumps(to_predict)).json()
+
+   classes = ["first","second","third"]
+
+   for idx,each_class in enumerate(classes): 
+      prediction_gauge.labels(order=each_class).set(result['predictions'][idx])
+   return json.dumps(result)
+
+
+
+
+@app.route('/predictions', methods=['POST'])
+# def predict_from_torch_serve():
+
+#    classes = ['tabby','tiger_cat','Egyptian_cat','lynx','tiger']
+
+#    to_predict = request.files['input']
+
+   
+#    result = requests.post("http://10.5.0.11:8080/predictions/resnet-18",data=to_predict).json() 
+   
+#    logger.info(f'prediction {json.dumps(result)}')
+#    #update prediction metric with all outputs 
+#    for each_class in classes: 
+#       prediction_gauge.labels(animal=each_class).set(result[each_class])
+# data
+#    return json.dumps(result)
+
+
+
+@app.route('/metrics')
+def metrics():
+   return Response(prometheus_client.generate_latest(),mimetype=str('text/plain; version=0.0.4; charset=utf-8'))
+
+
+
+
+
+if __name__ == '__main__':
+
+    app.run()

flask/app/train_and_predict.py

+
+
+
+"""
+Train ML model and define prediction function here 
+
+"""
+

flask/requirements.txt

+pandas
+numpy
+scikit-learn
+Flask 
+prometheus_client 
+requests
+Werkzeug==0.16.0

flask/run.py

+from app import app
+
+if __name__ == "__main__":
+    app.run(debug=True, host='0.0.0.0')
\ No newline at end of file

nginx/Dockerfile

+# Use the Nginx image
+FROM nginx
+
+# Remove the default nginx.conf
+RUN rm /etc/nginx/conf.d/default.conf
+
+# Replace with our own nginx.conf
+COPY nginx.conf /etc/nginx/conf.d/
\ No newline at end of file