How to reverse engineer communication encryption logic from the application?

This is an answer concerning the algorithm. It does not answer how to re-engineer your application.

A quick web search with the keywords "cctalk bnv" revealed (on the first page) this source code in a Github repository, licensed under the MIT License. Here it is, slightly formatted for better readability (the quality is disputable):

package com.vltgroup.ccTalk.commands;

public class BNVEncode {
  public static final int BNV_CODE_LENGTH = 6;
  
  public static boolean isValidBNVCode(byte[] BNVCode) {
    if (BNVCode == null || BNVCode.length != BNV_CODE_LENGTH) {
      return false;
    }
    for (byte b: BNVCode) {
      if (b != 0) {
        return true;
      }
    }
    return false;
  }
  
  private static final int rotatePlaces = 12;
  private static final int feedMaster = 99;
  private static final byte[] tapArray = { 7, 4, 5, 3, 1, 2, 3, 2, 6, 1 };
  
  public static void BNV_encrypt(byte[] secCode, byte[] data) {
    {
      int initXOR = ~(secCode[0] << 4 | secCode[4]);
      for (int i = 0; i < data.length; ++i)
        data[i] ^= initXOR;
    }
  
    for (int i = 0; i < data.length; ++i)
      if ((secCode[3] & (1 << (i & 0x03))) != 0) {
        byte t = data[i];
        data[i] = (byte) (((t & 0x01) << 7) | ((t & 0x02) << 5) | ((t & 0x04) << 3) | ((t & 0x08) << 1) |
                          ((t & 0x10) >> 1) | ((t & 0x20) >> 3) | ((t & 0x40) >> 5) | ((t & 0x80) >> 7));
      }

    for (int i = 0; i < rotatePlaces; ++i) {
      byte c1 = (data[data.length - 1] & 0x01) != 0 ? (byte) 128 : 0;

      for (int j = 0; j < data.length; ++j) {
        if ((data[j] & (1 << tapArray[(secCode[1] + j) % 10])) != 0)
          c1 ^= (byte) 128;
      }

      for (int j = 0; j < data.length; ++j) {
        byte c = (data[j] & 0x01) != 0 ? (byte) 128 : 0;
        if (((secCode[5] ^ feedMaster) & ( 1 << ((i + j) % 8))) != 0)
          c ^= (byte) 128;
      
        data[j] = (byte) (((data[j] & 0xFF) >>> 1) +  c1);
        c1 = c;
      }
    }

    {
      int finalXOR = (secCode[2] << 4 | secCode[2]);
      for (int i = 0; i < data.length; ++i)
        data[i] ^= finalXOR;
    }
  }
  
  public static void BNV_decrypt(byte[] secCode, byte[] data) {
    {
      int initXOR = (secCode[2] << 4 | secCode[2]);
      for (int i = 0; i < data.length; ++i)
        data[i] ^= initXOR;
    }

    for (int i = rotatePlaces - 1; i >= 0; --i) {
      byte c1 = (data[0] & 0x80) != 0 ? (byte) 1 : 0;

      for (int j = 0; j < data.length; ++j) {
        if ((data[j] & (1 << (tapArray[(secCode[1] + j) % 10] - 1))) != 0)
          c1 ^= 1;
      }

      for (int j = data.length-1; j >= 0; --j) {
        byte c = (data[j] & 0x80) !=0 ? (byte) 1 : 0;
        if (((secCode[5] ^ feedMaster) & ( 1 << ((i + j - 1) % 8))) != 0)
          c ^= 1;
        data[j] = (byte) ((data[j] << 1) + c1);
        c1 = c;
      }
    }

    for (int i = 0; i < data.length; ++i)
      if ((secCode[3] & (1 << (i & 0x03))) != 0) {
        byte t = data[i];
        data[i] = (byte) (((t & 0x01) << 7) | ((t & 0x02) << 5) | ((t & 0x04) << 3) | ((t & 0x08) << 1) |
                          ((t & 0x10) >> 1) | ((t & 0x20) >> 3) | ((t & 0x40) >> 5) | ((t & 0x80) >> 7)); 
      }

    {
      int finalXOR = ~(secCode[0] << 4 | secCode[4]);
      for (int i = 0; i < data.length; ++i)
        data[i] ^= finalXOR;
    }
  } 
}

This simple test application uses the Java class:

public class Test {
    private static byte[] KEY = { 1, 2, 3, 4, 5, 6, };

    public static void main(String[] args) {
        if (!BNVEncode.isValidBNVCode(KEY)) {
            System.out.println("The BNV key is not valid!");
            System.exit(0);
        }

        byte[] data1 = { 0x25, 0x02, 0x0F, };

        BNVEncode.BNV_encrypt(KEY, data1);
        System.out.printf("encrypted = 0x%02X, 0x%02X, 0x%02X\n", data1[0], data1[1], data1[2]);

        byte[] data2 = { 0x12, 0x19, (byte)0xB6, };

        BNVEncode.BNV_decrypt(KEY, data2);
        System.out.printf("decrypted = 0x%02X, 0x%02X, 0x%02X\n", data2[0], data2[1], data2[2]);
    }
}

The output shows that this is the applied algorithm:

$ java Test
encrypted = 0x12, 0x19, 0xB6
decrypted = 0x25, 0x02, 0x0F

The interesting detail is the format of the key. Each digit is stored in an own byte.