Restructure package (#622)

* Move around

* rename messaging to ipc

* More renames

* refactor visionipc ipc

* more movement

* compiles

* works well

* update workflow

* Update

* test fake

* fix names

* Fix test

* exclude library

* exclude from lint too

* Rm dir

* rm this wayu

* Try again

* mv logger

* delete old

* HAX

* Move logger down

* add warning abck

.github/workflows/tests.yml

@@ -40,8 +40,8 @@ jobs:
       run: |
         $RUN "export ${{ matrix.backend }}=1 && \
               scons ${{ matrix.flags }} -j$(nproc) && \
-              messaging/test_runner && \
-              visionipc/test_runner"
+              msgq/test_runner && \
+              msgq/visionipc/test_runner"
     - name: python tests
       run: $RUN_NAMED "${{ matrix.backend }}=1 coverage run -m unittest discover ."
     - name: Upload coverage

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+files: ^msgq/
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
@@ -21,7 +22,7 @@ repos:
         entry: cppcheck
         language: system
         types: [c++]
-        exclude: '^(messaging/msgq_tests.cc|messaging/test_runner.cc)'
+        exclude: '^(msgq/msgq_tests.cc|msgq/test_runner.cc)'
         args:
         - --error-exitcode=1
         - --inline-suppr

Dockerfile

@@ -37,11 +37,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 RUN pip3 install --break-system-packages --no-cache-dir pyyaml Cython scons pycapnp pre-commit ruff parameterized coverage numpy
 
-WORKDIR /project/
+WORKDIR /project/msgq/
 RUN cd /tmp/ && \
     git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git && \
     cd Catch2 && \
-    mv single_include/catch2/ /project/ && \
+    mv single_include/* /project/msgq/ && \
     cd .. \
     rm -rf Catch2
 
@@ -50,5 +50,5 @@ WORKDIR /project/msgq
 ENV PYTHONPATH=/project
 
 COPY . .
-RUN rm -rf .git && \
+RUN ls && rm -rf .git && \
     scons -c && scons -j$(nproc)

SConscript

@@ -1,40 +1,40 @@
 Import('env', 'envCython', 'arch', 'common')
 
 
-visionipc_dir = Dir('visionipc')
+visionipc_dir = Dir('msgq/visionipc')
 gen_dir = Dir('gen')
 
 
-# Build messaging
-
-
-messaging_objects = env.SharedObject([
-  'messaging/messaging.cc',
-  'messaging/event.cc',
-  'messaging/impl_zmq.cc',
-  'messaging/impl_msgq.cc',
-  'messaging/impl_fake.cc',
-  'messaging/msgq.cc',
+# Build msgq
+msgq_objects = env.SharedObject([
+  'msgq/ipc.cc',
+  'msgq/event.cc',
+  'msgq/impl_zmq.cc',
+  'msgq/impl_msgq.cc',
+  'msgq/impl_fake.cc',
+  'msgq/msgq.cc',
 ])
-messaging = env.Library('messaging', messaging_objects)
-messaging_python = envCython.Program('messaging/messaging_pyx.so', 'messaging/messaging_pyx.pyx', LIBS=envCython["LIBS"]+[messaging, "zmq", common])
-
+msgq = env.Library('msgq', msgq_objects)
+msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq, "zmq", common])
 
 # Build Vision IPC
-vipc_files = ['ipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc']
+vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc']
 vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files]
 
 if arch == "larch64":
   vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_ion.cc']
 else:
   vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_cl.cc']
 
+print(f'Building Vision IPC with {vipc_sources}')
 vipc_objects = env.SharedObject(vipc_sources)
+print(f'Building Vision IPC with {vipc_objects}')
+
 visionipc = env.Library('visionipc', vipc_objects)
 
 
 vipc_frameworks = []
-vipc_libs = envCython["LIBS"] + [visionipc, messaging, common, "zmq"]
+vipc_libs = envCython["LIBS"] + [visionipc, msgq, common, "zmq"]
 if arch == "Darwin":
   vipc_frameworks.append('OpenCL')
 else:
@@ -43,9 +43,9 @@ envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.
                   LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks)
 
 if GetOption('extras'):
-  env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging, common])
-  env.Program('visionipc/test_runner',
-             ['visionipc/test_runner.cc', 'visionipc/visionipc_tests.cc'],
+  env.Program('msgq/test_runner', ['msgq/test_runner.cc', 'msgq/msgq_tests.cc'], LIBS=[msgq, common])
+  env.Program(f'{visionipc_dir.abspath}/test_runner',
+             [f'{visionipc_dir.abspath}/test_runner.cc', f'{visionipc_dir.abspath}/visionipc_tests.cc'],
               LIBS=['pthread'] + vipc_libs, FRAMEWORKS=vipc_frameworks)
 
-Export('visionipc', 'messaging', 'messaging_python')
+Export('visionipc', 'msgq', 'msgq_python')

SConstruct

@@ -11,7 +11,8 @@ if platform.system() == "Darwin":
 common = ''
 
 cpppath = [
-  f"#/../",
+  f"#/",
+  '#msgq/',
   '/usr/lib/include',
   '/opt/homebrew/include',
   sysconfig.get_paths()['include'],

msgq/.gitignore

@@ -0,0 +1 @@
+ipc_pyx.cpp

messaging/__init__.py → msgq/__init__.py

@@ -1,12 +1,12 @@
 # must be built with scons
-from msgq.messaging.messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
                                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
-from msgq.messaging.messaging_pyx import MultiplePublishersError, MessagingError
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
 
 from typing import Optional, List
 
 assert MultiplePublishersError
-assert MessagingError
+assert IpcError
 assert toggle_fake_events
 assert set_fake_prefix
 assert get_fake_prefix

messaging/event.cc → msgq/event.cc

@@ -13,7 +13,7 @@
 #include <sys/mman.h>
 #include <sys/stat.h>
 
-#include "msgq/messaging/event.h"
+#include "msgq/event.h"
 
 #ifndef __APPLE__
 #include <sys/eventfd.h>

messaging/impl_fake.cc → msgq/impl_fake.cc

@@ -1,4 +1,4 @@
-#include "msgq/messaging/impl_fake.h"
+#include "msgq/impl_fake.h"
 
 void FakePoller::registerSocket(SubSocket *socket) {
   this->sockets.push_back(socket);

messaging/impl_fake.h → msgq/impl_fake.h

@@ -11,8 +11,8 @@
 #include <fcntl.h>
 #include <unistd.h>
 
-#include "msgq/messaging/messaging.h"
-#include "msgq/messaging/event.h"
+#include "msgq/ipc.h"
+#include "msgq/event.h"
 
 template<typename TSubSocket>
 class FakeSubSocket: public TSubSocket {

messaging/impl_msgq.cc → msgq/impl_msgq.cc

@@ -5,7 +5,7 @@
 #include <csignal>
 #include <cerrno>
 
-#include "msgq/messaging/impl_msgq.h"
+#include "msgq/impl_msgq.h"
 
 
 volatile sig_atomic_t msgq_do_exit = 0;

messaging/impl_msgq.h → msgq/impl_msgq.h

@@ -3,8 +3,8 @@
 #include <string>
 #include <vector>
 
-#include "msgq/messaging/messaging.h"
-#include "msgq/messaging/msgq.h"
+#include "msgq/ipc.h"
+#include "msgq/msgq.h"
 
 #define MAX_POLLERS 128
 

messaging/impl_zmq.cc → msgq/impl_zmq.cc

@@ -5,7 +5,7 @@
 #include <cerrno>
 #include <unistd.h>
 
-#include "msgq/messaging/impl_zmq.h"
+#include "msgq/impl_zmq.h"
 
 //FIXME: This is a hack to get the port number from the socket name, might have collisions
 static int get_port(std::string endpoint) {

messaging/impl_zmq.h → msgq/impl_zmq.h

@@ -4,7 +4,7 @@
 #include <string>
 #include <vector>
 
-#include "msgq/messaging/messaging.h"
+#include "msgq/ipc.h"
 
 #define MAX_POLLERS 128
 

messaging/messaging.cc → msgq/ipc.cc

@@ -1,10 +1,10 @@
 #include <cassert>
 #include <iostream>
 
-#include "msgq/messaging/messaging.h"
-#include "msgq/messaging/impl_zmq.h"
-#include "msgq/messaging/impl_msgq.h"
-#include "msgq/messaging/impl_fake.h"
+#include "msgq/ipc.h"
+#include "msgq/impl_zmq.h"
+#include "msgq/impl_msgq.h"
+#include "msgq/impl_fake.h"
 
 #ifdef __APPLE__
 const bool MUST_USE_ZMQ = true;

messaging/messaging.pxd → msgq/ipc.pxd

@@ -6,7 +6,7 @@ from libcpp.vector cimport vector
 from libcpp cimport bool
 
 
-cdef extern from "msgq/messaging/impl_fake.h":
+cdef extern from "msgq/impl_fake.h":
   cdef cppclass Event:
     @staticmethod
     int wait_for_one(vector[Event], int) except +
@@ -34,7 +34,7 @@ cdef extern from "msgq/messaging/impl_fake.h":
     Event recv_ready()
 
 
-cdef extern from "msgq/messaging/messaging.h":
+cdef extern from "msgq/ipc.h":
   cdef cppclass Context:
     @staticmethod
     Context * create()

messaging/messaging_pyx.pyx → msgq/ipc_pyx.pyx

@@ -10,22 +10,22 @@ from libc.string cimport strerror
 from cython.operator import dereference
 
 
-from .messaging cimport Context as cppContext
-from .messaging cimport SubSocket as cppSubSocket
-from .messaging cimport PubSocket as cppPubSocket
-from .messaging cimport Poller as cppPoller
-from .messaging cimport Message as cppMessage
-from .messaging cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle
+from .ipc cimport Context as cppContext
+from .ipc cimport SubSocket as cppSubSocket
+from .ipc cimport PubSocket as cppPubSocket
+from .ipc cimport Poller as cppPoller
+from .ipc cimport Message as cppMessage
+from .ipc cimport Event as cppEvent, SocketEventHandle as cppSocketEventHandle
 
 
-class MessagingError(Exception):
+class IpcError(Exception):
   def __init__(self, endpoint=None):
     suffix = f"with {endpoint.decode('utf-8')}" if endpoint else ""
     message = f"Messaging failure {suffix}: {strerror(errno.errno).decode('utf-8')}"
     super().__init__(message)
 
 
-class MultiplePublishersError(MessagingError):
+class MultiplePublishersError(IpcError):
   pass
 
 
@@ -170,7 +170,7 @@ cdef class SubSocket:
     self.is_owner = True
 
     if self.socket == NULL:
-      raise MessagingError
+      raise IpcError
 
   def __dealloc__(self):
     if self.is_owner:
@@ -190,7 +190,7 @@ cdef class SubSocket:
       if errno.errno == errno.EADDRINUSE:
         raise MultiplePublishersError(endpoint)
       else:
-        raise MessagingError(endpoint)
+        raise IpcError(endpoint)
 
   def setTimeout(self, int timeout):
     self.socket.setTimeout(timeout)
@@ -219,7 +219,7 @@ cdef class PubSocket:
   def __cinit__(self):
     self.socket = cppPubSocket.create()
     if self.socket == NULL:
-      raise MessagingError
+      raise IpcError
 
   def __dealloc__(self):
     del self.socket
@@ -231,7 +231,7 @@ cdef class PubSocket:
       if errno.errno == errno.EADDRINUSE:
         raise MultiplePublishersError(endpoint)
       else:
-        raise MessagingError(endpoint)
+        raise IpcError(endpoint)
 
   def send(self, bytes data):
     length = len(data)
@@ -241,7 +241,7 @@ cdef class PubSocket:
       if errno.errno == errno.EADDRINUSE:
         raise MultiplePublishersError
       else:
-        raise MessagingError
+        raise IpcError
 
   def all_readers_updated(self):
     return self.socket.all_readers_updated()

messaging/msgq.cc → msgq/msgq.cc

@@ -23,7 +23,7 @@
 
 #include <stdio.h>
 
-#include "msgq/messaging/msgq.h"
+#include "msgq/msgq.h"
 
 void sigusr2_handler(int signal) {
   assert(signal == SIGUSR2);

messaging/msgq_tests.cc → msgq/msgq_tests.cc

@@ -1,5 +1,5 @@
 #include "catch2/catch.hpp"
-#include "msgq/messaging/msgq.h"
+#include "msgq/msgq.h"
 
 TEST_CASE("ALIGN")
 {