Spresenseをつかったカメラセンサ（０／１判定）

配布物チェック空振り防止システム（https://protopedia.net/prototype/4713）に使用した、Spresense＋カメラモジュールで印刷物等があるかないかを確認するシステム。
バッテリで動作させる事を前提とするため、１日１回の動作とし、Wi-Fiが使用できない箇所で使うためLTE回線が必要となることからSpresenseを採用。
ここでは、カメラで撮影したデータから判定に必要となるデータを抽出し、Kintoneサーバにアップロードまでを紹介します。（中継にSoracom Beamを使用しています）

Spresene メインボード
Spresense LTE拡張ボード
Spresense HDRカメラボード
SIMカード（ソラコム）
SDカード（Koxia 16GB)

SpresenseメインボードにLTE拡張ボードとHDRカメラボードを接続。
LTE拡張ボードにSIMカード、及びSDカードを挿入。

その他の外部機器との接続・配線は行っていません。

画像フォーマットをYUV422 format形式で取得。QVGAで取得後、データのノイズとデータ処理量を減らすため、画像サイズを縮小しています。
　＊ソース上はJPEGで画像撮影し、SDカードに保存してから上記の撮影を行っています。

画像の１画素ごとの階調データを取り出し、階調データの値MAX値・MIN値・平均値・分散値を算出。さらにコントラスト（MAX値/MIN値）、均一性（平均値/分散値）を求めます。
　データ取得については以下のようなイメージとなります。
　
　このデータを元に、配布物の有無を判定するのですが、設置環境（環境の明るさ、影の入り具合等）によって閾値が異なるので、配布物があるとき・ないときを何度かデータ取得し、分析する必要があります。
　今回はたまたまコントラスト値だけを閾値として実用化できましたので、それで判定しています。

＊ソースコードでは検討用にSDカードにもデータを保存しています。

測定したデータをKintoneにアップロードします。アップロードに際して、Soracom Beamを使用し、Soracom Beam上で必要なヘッダ等を付与しています。

(設定不可の項目も記載しています）

設定を追加する→HTTPエントリポイント　を選択
設定名は適当に・・
エントリポイント
　プロトコル：HTTP
　ホスト名：beam.soracom.io
　ポート番号：8888
　パス：/record.json
転送先
　プロトコル：HTTPS
　ホスト名：<Kintoneのドメイン>.cybozu.com
　ポート番号：443
　パス：/k/v1/record.json
カスタムヘッダ
　アクション：追加
　ヘッダ名：X-Cybozu-API-Token
　値：＜Kintoneで得られるAPIトークン＞

受け側として以下のフィールドコードを持つレコードを設定します。具体的な設定方法等は省略。
　　timestamp、Average、Sigma、MAX、MIN、result

消費電力を削減するため、Kintoneへのアップロードが完了したら、DeepSleepとします。
　１日に１回、おおむね決まった時刻に動作させるため、LTE接続時に取得したタイムスタンプから、翌日の設定時刻までの時間（秒）を算出し、その値をDeepSleepで設定するインターバルタイムとしています。

＊サンプルコードのコメント文が部分的に残っています


Spresense_Camera_to_Kintone.ino
#include <SDHCI.h>
#include <stdio.h> 

#include <Camera.h>

#include <LowPower.h>
#include <RTC.h>

#include <ArduinoHttpClient.h>
#include <LTE.h>
#include <LTEClient.h>

#define BAUDRATE                (115200)
#define TOTAL_PICTURE_COUNT     (10)

SDClass  theSD;
int take_picture_count = 0;

CamErr err;

// APN name
#define APP_LTE_APN "soracom.io" // replace your APN
#define APP_LTE_USER_NAME "sora" // replace with your username
#define APP_LTE_PASSWORD  "sora" // replace with your password

// APN IP type
#define APP_LTE_IP_TYPE (LTE_NET_IPTYPE_V4V6) // IP : IPv4v6
// APN authentication type
#define APP_LTE_AUTH_TYPE (LTE_NET_AUTHTYPE_CHAP) // Authentication : CHAP
#define APP_LTE_RAT (LTE_NET_RAT_CATM) // RAT : LTE-M (LTE Cat-M1)

char server[] = "beam.soracom.io";
char postPath[] = "/record.json";
int port = 8888; 

uint32_t IntervalSec;
uint8_t SetHour = 11;   //起動時刻（時間）
uint8_t SetMinute = 10;  //起動時刻（分）


LTE lteAccess;
LTEClient client;


/**
 * Print error message
 */

void printError(enum CamErr err)
{
  Serial.print("Error: ");
  switch (err)
    {
      case CAM_ERR_NO_DEVICE:
        Serial.println("No Device");
        break;
      case CAM_ERR_ILLEGAL_DEVERR:
        Serial.println("Illegal device error");
        break;
      case CAM_ERR_ALREADY_INITIALIZED:
        Serial.println("Already initialized");
        break;
      case CAM_ERR_NOT_INITIALIZED:
        Serial.println("Not initialized");
        break;
      case CAM_ERR_NOT_STILL_INITIALIZED:
        Serial.println("Still picture not initialized");
        break;
      case CAM_ERR_CANT_CREATE_THREAD:
        Serial.println("Failed to create thread");
        break;
      case CAM_ERR_INVALID_PARAM:
        Serial.println("Invalid parameter");
        break;
      case CAM_ERR_NO_MEMORY:
        Serial.println("No memory");
        break;
      case CAM_ERR_USR_INUSED:
        Serial.println("Buffer already in use");
        break;
      case CAM_ERR_NOT_PERMITTED:
        Serial.println("Operation not permitted");
        break;
      default:
        break;
    }
}

/**
 * Callback from Camera library when video frame is captured.
 */

void CamCB(CamImage img)
{

  /* Check the img instance is available or not. */

  if (img.isAvailable())
    {

      /* If you want RGB565 data, convert image data format to RGB565 */

      img.convertPixFormat(CAM_IMAGE_PIX_FMT_RGB565);

      /* You can use image data directly by using getImgSize() and getImgBuff().
       * for displaying image to a display, etc. */

      Serial.print("Image data size = ");
      Serial.print(img.getImgSize(), DEC);
      Serial.print(" , ");

      Serial.print("buff addr = ");
      Serial.print((unsigned long)img.getImgBuff(), HEX);
      Serial.println("");
    }
  else
    {
      Serial.println("Failed to get video stream image");
    }
}

void printClock(RtcTime &rtc)
{
  printf("%04d/%02d/%02d %02d:%02d:%02d\n",
         rtc.year(), rtc.month(), rtc.day(),
         rtc.hour(), rtc.minute(), rtc.second());
}

String readFromSerial() {
  /* Read String from serial monitor */
  String str;
  int  read_byte = 0;
  while (true) {
    if (Serial.available() > 0) {
      read_byte = Serial.read();
      if (read_byte == '\n' || read_byte == '\r') {
        Serial.println("");
        break;
      }
      Serial.print((char)read_byte);
      str += (char)read_byte;
    }
  }
  return str;
}

void readApnInformation(char apn[], LTENetworkAuthType *authtype,
                       char user_name[], char password[]) {
  /* Set APN parameter to arguments from readFromSerial() */

  String read_buf;

  while (strlen(apn) == 0) {
    Serial.print("Enter Access Point Name:");
    readFromSerial().toCharArray(apn, LTE_NET_APN_MAXLEN);
  }

  while (true) {
    Serial.print("Enter APN authentication type(CHAP, PAP, NONE):");
    read_buf = readFromSerial();
    if (read_buf.equals("NONE") == true) {
      *authtype = LTE_NET_AUTHTYPE_NONE;
    } else if (read_buf.equals("PAP") == true) {
      *authtype = LTE_NET_AUTHTYPE_PAP;
    } else if (read_buf.equals("CHAP") == true) {
      *authtype = LTE_NET_AUTHTYPE_CHAP;
    } else {
      /* No match authtype */
      Serial.println("No match authtype. type at CHAP, PAP, NONE.");
      continue;
    }
    break;
  }

  if (*authtype != LTE_NET_AUTHTYPE_NONE) {
    while (strlen(user_name)== 0) {
      Serial.print("Enter username:");
      readFromSerial().toCharArray(user_name, LTE_NET_USER_MAXLEN);
    }
    while (strlen(password) == 0) {
      Serial.print("Enter password:");
      readFromSerial().toCharArray(password, LTE_NET_PASSWORD_MAXLEN);
    }
  }

  return;
}


/**
 * @brief Initialize camera
 */
void setup()
{
  /* Open serial communications and wait for port to open */

  Serial.begin(BAUDRATE);
  while (!Serial)
    {
      ; /* wait for serial port to connect. Needed for native USB port only */
    }

  /* Deep Speep Set Up */
  RTC.begin();
  srand(time(NULL));

  LowPower.begin();
  LowPower.clockMode(CLOCK_MODE_32MHz);
  bootcause_e bc = LowPower.bootCause(); /* get boot cause */


  /* Initialize SD */
  while (!theSD.begin()) 
    {
      /* wait until SD card is mounted. */
      Serial.println("Insert SD card.");
    }

  /* begin() without parameters means that
   * number of buffers = 1, 30FPS, QVGA, YUV 4:2:2 format */

  char pictname[16] = {0};
  char filenameD[16] = {0};

  Serial.println("Prepare camera");
  err = theCamera.begin();
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }

  /* Auto white balance configuration */

  Serial.println("Set Auto white balance parameter");
  err = theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_DAYLIGHT);
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }
 
  /* Set parameters about6still picture.
   * In the following case, QUADVGA and JPEG.
   */

  Serial.println("Set still picture format");
  err = theCamera.setStillPictureImageFormat(
     CAM_IMGSIZE_QUADVGA_H,
     CAM_IMGSIZE_QUADVGA_V,
     CAM_IMAGE_PIX_FMT_JPG
     );
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }

//  take_picture_count++;
  Serial.println("call takePicture()");
  CamImage img = theCamera.takePicture();

      /* Check availability of the img instance. */
      /* If any errors occur, the img is not available. */

  if (img.isAvailable())
    {
      // File Count //
      File myFile0 = theSD.open("counter.txt");
      uint8_t count_read[20] = {0};
      uint8_t file_count[5] = {0};
      if (myFile0) {
        int i_seek = 0;
        while (myFile0.available()){
          count_read[i_seek] = myFile0.read();
          i_seek++;
        }
        myFile0.close();
        int digit_count =0;
        for (int z=0; z<i_seek; z++){
            //Serial.println(file_count[z]);
            //先頭4ビットが3のとき,数字に割り当てる//
            if ((count_read[z] & 0b11110000) == 0b00110000) { 
                  file_count[z] = count_read[z] & 0b00001111;
                  digit_count ++;
            }
        }
        int counter_tmp;
        for (int y=0; y<digit_count; y++){
            counter_tmp = file_count[y];
            for (int x=0; x<digit_count-y-1; x++){
             counter_tmp = counter_tmp * 10;
            }
            take_picture_count = take_picture_count + counter_tmp;
        }
        take_picture_count ++;

        theSD.remove("counter.txt");
        File myFileZ = theSD.open("counter.txt", FILE_WRITE);
        
        myFileZ.println(take_picture_count);
        myFileZ.close();
      }
      else {
        Serial.println("File Open Error");
        take_picture_count = 0;
        File myFileZ = theSD.open("counter.txt", FILE_WRITE);
        myFileZ.print(0);
        myFileZ.close();
      }

          /* Create file name */
      char filename[16] = {0};

      sprintf(filename, "PICT%03d.JPG", take_picture_count);
      sprintf(pictname, "PICT%03d.JPG", take_picture_count);
      //sprintf(filename, "PICT123.JPG");

      Serial.print("Save taken picture as ");
      Serial.print(filename);
      Serial.println("");

          /* Remove the old file with the same file name as new created file,
           * and create new file.
           */
      theSD.remove(filename);
      File myFile = theSD.open(filename, FILE_WRITE);
      myFile.write(img.getImgBuff(), img.getImgSize());
      myFile.close();
      Serial.println("Picture File Saved.");
    }
  else
    {
          /* The size of a picture may exceed the allocated memory size.
           * Then, allocate the larger memory size and/or decrease the size of a picture.
           * [How to allocate the larger memory]
           * - Decrease jpgbufsize_divisor specified by setStillPictureImageFormat()
           * - Increase the Memory size from Arduino IDE tools Menu
           * [How to decrease the size of a picture]
           * - Decrease the JPEG quality by setJPEGQuality()
           */

      Serial.println("Failed to take picture");
    }

  theCamera.end();
  
  sleep(1);

  Serial.println("Start analysis");

  float data_average;
  float data_sigma;
  float data_max = 0;
  float data_min = 255;
  char massage[36];

  //Serial.println("Prepare camera");
  err = theCamera.begin();
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }

  /* Auto white balance configuration */
  //Serial.println("Set Auto white balance parameter");
  err = theCamera.setAutoWhiteBalanceMode(CAM_WHITE_BALANCE_DAYLIGHT);
  if (err != CAM_ERR_SUCCESS)
    {
      printError(err);
    }
  /* Set parameters about still picture.*/
  /*QVGA and YUV422 format. */
  //Serial.println("Change Format and call takePicture()");
  err = theCamera.setStillPictureImageFormat(320,240,CAM_IMAGE_PIX_FMT_YUV422);
  if (err != CAM_ERR_SUCCESS)
    { 
      printError(err);
    }
  CamImage img_2 = theCamera.takePicture();
  if (img.isAvailable())
    {
      CamImage res_img;
      int x_size = 80;
      int y_size = 60;
      
      err = img_2.resizeImageByHW(res_img,x_size,y_size);
      if (!res_img.isAvailable()){
          printError(err);
          }

      uint16_t* imgbuf = (uint16_t*)res_img.getImgBuff();
      uint8_t *grayImg;
      int  Imagesize =  x_size * y_size;

      if ((grayImg = (uint8_t *)malloc(Imagesize)) == NULL) {
          printf("Failed to apply for black memory...\r\n");
          while (1);
      }

      for (int n=0; n<Imagesize; ++n){
        grayImg[n] = (uint8_t)(((imgbuf[n] & 0xf000) >> 8) 
                         | ((imgbuf[n] & 0x00f0) >> 4));
        } 

      uint32_t data_sum = 0;
      for (int n=0; n<Imagesize; ++n){
        data_sum = data_sum + grayImg[n];
      }
      data_average = data_sum/Imagesize;

      uint32_t data_sum2 = 0;
      for (int n=0; n<Imagesize; ++n){
        data_sum2 = data_sum2 + grayImg[n]*grayImg[n];
      }
      float data_ave2 = data_average*data_average;      
      float data_var = data_sum2/Imagesize - data_ave2;
      data_sigma = sqrt(data_var);
      float data_cv = data_sigma/data_average;

      for (int n=0; n<Imagesize; ++n){
        if (grayImg[n] > data_max) {
          data_max = grayImg[n];
        }
      }
      for (int n=0; n<Imagesize; ++n){
        if (grayImg[n] < data_min) {
          data_min = grayImg[n];
        }
      }
      float data_contrast;
        data_contrast = (data_max+1)/(data_min+1);
      
      if (data_contrast > 3){
        sprintf(massage,"配布物あり");
      }
      else{
        sprintf(massage,"配布物なし");
      }

      Serial.println("<<Analysis result>>");
      Serial.print("average = ");
      Serial.print(data_average);
      Serial.print(", sigma = ");
      Serial.print(data_sigma);
      Serial.print(", CV = ");
      Serial.print(data_cv);
      Serial.print(", MAX = ");
      Serial.print(data_max);
      Serial.print(", MIN = ");
      Serial.print(data_min);
      Serial.print(", CONTRAST = ");
      Serial.println(data_contrast);
      Serial.print(", RESULT : ");
      Serial.println(massage);


      sprintf(filenameD, "DAT%03d.csv", take_picture_count);

      Serial.print("Save data as ");
      Serial.print(filenameD);
      Serial.println("");

          /* Remove the old file with the same file name as new created file,
           * and create new file.
           */
      theSD.remove(filenameD);
      File myFileD = theSD.open(filenameD, FILE_WRITE);
      myFileD.println("average,sigma,CV,MAX,MIN,Contrast");

      myFileD.print(data_average);
      myFileD.print(",");
      myFileD.print(data_sigma);
      myFileD.print(",");
      myFileD.print(data_cv);
      myFileD.print(",");
      myFileD.print(data_max);
      myFileD.print(",");
      myFileD.print(data_min);
      myFileD.print(",");
      myFileD.println(data_contrast);

      myFileD.close();

      free(grayImg);
      grayImg = NULL;

    }
  else
    {
      Serial.println("Failed to take picture");
    }

  char apn[LTE_NET_APN_MAXLEN] = APP_LTE_APN;
  LTENetworkAuthType authtype = APP_LTE_AUTH_TYPE;
  char user_name[LTE_NET_USER_MAXLEN] = APP_LTE_USER_NAME;
  char password[LTE_NET_PASSWORD_MAXLEN] = APP_LTE_PASSWORD;
  Serial.println("Starting secure HTTP client.");

  /* Set if Access Point Name is empty */
  if (strlen(APP_LTE_APN) == 0) {
    Serial.println("This sketch doesn't have a APN information.");
    readApnInformation(apn, &authtype, user_name, password);
  }
  Serial.println("=========== APN information ===========");
  Serial.print("Access Point Name  : ");
  Serial.println(apn);
  Serial.print("Authentication Type: ");
  Serial.println(authtype == LTE_NET_AUTHTYPE_CHAP ? "CHAP" :
                 authtype == LTE_NET_AUTHTYPE_NONE ? "NONE" : "PAP");
  if (authtype != LTE_NET_AUTHTYPE_NONE) {
    Serial.print("User Name          : ");
    Serial.println(user_name);
    Serial.print("Password           : ");
    Serial.println(password);
  }
  while (true) {
    /* Power on the modem and Enable the radio function. */
    if (lteAccess.begin() != LTE_SEARCHING) {
      Serial.println("Could not transition to LTE_SEARCHING.");
      Serial.println("Please check the status of the LTE board.");
      for (;;) {
        sleep(1);
      }
    }
    /* The connection process to the APN will start.
     * If the synchronous parameter is false,
     * the return value will be returned when the connection process is started.
     */
    if (lteAccess.attach(APP_LTE_RAT,
                         apn,
                         user_name,
                         password,
                         authtype,
                         APP_LTE_IP_TYPE) == LTE_READY) {
      Serial.println("attach succeeded.");
      break;
    }
    /* If the following logs occur frequently, one of the following might be a cause:
     * - APN settings are incorrect
     * - SIM is not inserted correctly
     * - If you have specified LTE_NET_RAT_NBIOT for APP_LTE_RAT,
     *   your LTE board may not support it.
     * - Rejected from LTE network
     */
    Serial.println("An error has occurred. Shutdown and retry the network attach process after 1 second.");
    lteAccess.shutdown();
    sleep(1);
  }


  unsigned long currentTime;
  while(0 == (currentTime = lteAccess.getTime())) {
    sleep(1);
  }
  RtcTime rtc(currentTime);
  char timestamp[64];
  sprintf(timestamp,"%02d/%02d %02d:%02d",
         rtc.month(), rtc.day(),
         rtc.hour(), rtc.minute());

  uint32_t currentsec = currentTime % 86400; 
  uint32_t SetSecond = SetHour *3600 + SetMinute * 60;
  IntervalSec = 86400 - currentsec + SetSecond;
  if (IntervalSec > 86400){
    IntervalSec = IntervalSec - 86400;
  }

  Serial.print("timestamp = ");
  Serial.println(timestamp);

  Serial.println("making POST request at lteclient");
  char sendData[4096];
  sprintf(sendData,"{\"app\":\"4\",\"record\":{\"Average\":{\"value\":\"%f\"},\"Sigma\":{\"value\":\"%f\"},\"MAX\":{\"value\":\"%f\"},\"MIN\":{\"value\":\"%f\"},\"timestamp\":{\"value\":\"%s\"},\"result\":{\"value\":\"%s\"}}}",data_average,data_sigma,data_max,data_min,timestamp,massage);
  String contentType = "application/json";

  HttpClient http(client, server, port);
  http.beginRequest();
  http.post(postPath, contentType, sendData);
  http.endRequest();
  // read the status code and body of the response
  int statusCode = http.responseStatusCode();
  String response = http.responseBody();

  Serial.print("Status code: ");
  Serial.println(statusCode);
  Serial.print("Response: ");
  Serial.println(response);
}

void loop()
{
  Serial.println("Sleep mode.");
  Serial.print("Sleep time =");
  Serial.println(IntervalSec);  

  LowPower.deepSleep(IntervalSec);
}