Error in Repeated Record

[GLStephen]

Environments

• fluentd version: 17
• plugin version: 3.1.0

Configuration

######################################################
## RESPONSE
############################ #########################
<match *>
  @type bigquery_load
  @id bq_response

  <buffer time>
      @type file
      #path /var/log/fluent-buffer/buf/bigquery.*.buffer
      flush_at_shutdown true
      timekey_use_utc
      timekey 1d
  </buffer>

  auth_method json_key
  json_key /service-key.json

  time_partitioning_type day
  ignore_unknown_values true
  max_bad_records 10

  project gcpProject
  dataset bqDataset
  auto_create_table false

  # table specific info
  table loadBidResponse$%Y%m%d
  schema_path /fluentd/etc/schemas/loadBidResponse.json
</match>

Expected Behavior

Data is loaded according to schema.

Actual Behavior

The two following errors occur. I've broken up the lines a bit to make the sections more clear.

2024-07-11 17:35:13 +0000 [error]: #0 fluent/log.rb:404:error: unexpected error on reading data host="10.0.21.5" port=59364 error_class=NoMethodError error="undefined method `each' for \"td.com\":String\n\n        record.each do |key, value|\n              ^^^^^"

with this schema:

... other schema ...
{
    "fields": [
     {
      "name": "domain",
      "type": "STRING"
     }
    ],
    "mode": "REPEATED",
    "name": "advertiserDomains",
    "type": "RECORD"
   },
... other schema ...

seemingly related to this log:

Log (if you have)

I added some linebreaks to highlight the area with the issue.

2024-07-11 17:06:54 +0000 [error]: #0 [bq_response] format error record={"bidderCode"=>"ix", "width"=>320, "height"=>50, "statusMessage"=>"Bid available", "adId"=>"xxxxxxx", "requestId"=>"16f2728244731698", "transactionId"=>"xxxxxxxxxxxxxx", "adUnitId"=>"208e566a-61aa-4084-b7bd-4aa35ec1c622", "auctionId"=>"c57db871-6079-4de5-8e68-e7e39d698fb8", "mediaType"=>"banner", "source"=>"client", "cpm"=>0.43, "netRevenue"=>true, "currency"=>"USD", "creativeId"=>"27402600", "ad"=>"", "ttl"=>600, "meta"=>{
"networkId"=>85, 
"brandId"=>350488, 
"brandName"=>"Nestle - Nespresso", 
"advertiserDomains"=>
["nespresso.com"]
}
, "metrics"=>{"userId.init.consent"=>[0], "userId.mod.init"=>[1, 0, 0, 1, 0, 0, 3, 0], "userId.mods.33acrossId.init"=>[1], "userId.mods.amxId.init"=>[0], "userId.mods.criteo.init"=>[0], "userId.mods.fabrickId.init"=>[1], "userId.mods.id5Id.init"=>[0], "userId.mods.identityLink.init"=>[0], "userId.mods.sharedId.init"=>[3], "userId.mods.unifiedId.init"=>[0], "userId.init.modules"=>[7], "requestBids.usp"=>1, "requestBids.gdpr"=>0, "requestBids.userId"=>3, "requestBids.validate"=>1, "requestBids.makeRequests"=>8, "adapter.client.validate"=>0, "adapters.client.ix.validate"=>0, "adapter.client.buildRequests"=>3, "adapters.client.ix.buildRequests"=>3, "adapter.client.total"=>370, "adapters.client.ix.total"=>370, "adapter.client.net"=>363, "adapters.client.ix.net"=>363, "adapter.client.interpretResponse"=>1, "adapters.client.ix.interpretResponse"=>1, "addBidResponse.validate"=>0, "addBidResponse.total"=>2}, "adapterCode"=>"ix", "originalCpm"=>0.43, "originalCurrency"=>"USD", "responseTimestamp"=>1720717613381, "requestTimestamp"=>1720717613012, "bidder"=>"ix", "adUnitCode"=>"Aqua-Calc_Adhesive", "timeToRespond"=>369, "pbLg"=>"0.00", "pbMg"=>"0.40", "pbHg"=>"0.43", "pbAg"=>"0.40", "pbDg"=>"0.43", "pbCg"=>"0.43", "size"=>"320x50", "adserverTargeting"=>"", "auctionEnded"=>false, "utm_content"=>"", "guid6"=>"84a51f", "user_agent"=>"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1", "c_script_type"=>"default", "utm_source"=>"", "c_slot2"=>"", "c_slot3"=>"", "api_server"=>"", "pw_version"=>"4.0.1", "pbjs_version"=>"8.47.0", "c_slot1"=>"", "c_host"=>"s", "utm_campaign"=>"", "utm_term"=>"", "debug"=>false, "api_hour"=>"17", "api_ver"=>"v9.1", "account_id"=>326, "session_id"=>"a6af0cb5-aa9e-452f-aa7a-8de9f59196a8", "api_timestamp"=>"2024-07-11T17:06:53.778804173Z", "c_slot4"=>"", "activation_id"=>"f476cb75-1464-4302-bbee-18905d164a19", "utm_medium"=>""}

schema=#<Fluent::BigQuery::RecordSchema:0x00007f68a177de48 
@name="record", 
@mode=:nullable, 
@fields={"api_timestamp"=>#<Fluent::BigQuery::TimestampFieldSchema:0x00007f68a178dd20 @name="api_timestamp", @mode=:required>, "api_hour"=>#<Fluent::BigQuery::IntegerFieldSchema:0x00007f68a178d230 @name="api_hour", @mode=:required>, "account_id"=>#<Fluent::BigQuery::IntegerFieldSchema:0x00007f68a178c8f8 @name="account_id", @mode=:required>, "guid6"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a178c150 @name="guid6", @mode=:required>, "auctionId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17937e8 @name="auctionId", @mode=:nullable>, "bidderCode"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a1792af0 @name="bidderCode", @mode=:nullable>, "responseTimestamp"=>#<Fluent::BigQuery::IntegerFieldSchema:0x00007f68a1792258 @name="responseTimestamp", @mode=:nullable>, "requestTimestamp"=>#<Fluent::BigQuery::IntegerFieldSchema:0x00007f68a1791998 @name="requestTimestamp", @mode=:nullable>, "cpm"=>#<Fluent::BigQuery::FloatFieldSchema:0x00007f68a1790db8 @name="cpm", @mode=:nullable>, "bidder"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a1797960 @name="bidder", @mode=:nullable>, "adUnitCode"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a179a7a0 @name="adUnitCode", @mode=:nullable>, "adId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17997d8 @name="adId", @mode=:nullable>, "statusMessage"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17990d0 @name="statusMessage", @mode=:nullable>, "width"=>#<Fluent::BigQuery::IntegerFieldSchema:0x00007f68a179f5c0 @name="width", @mode=:nullable>, "height"=>#<Fluent::BigQuery::IntegerFieldSchema:0x00007f68a179ed78 @name="height", @mode=:nullable>, "source"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a179e530 @name="source", @mode=:nullable>, "api_ver"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a179d3d8 @name="api_ver", @mode=:nullable>, "api_server"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a179cb18 @name="api_server", @mode=:nullable>, "user_device_type"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a179c4b0 @name="user_device_type", @mode=:nullable>, "utm_source"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a179c140 @name="utm_source", @mode=:nullable>, "utm_medium"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a3d78 @name="utm_medium", @mode=:nullable>, "utm_campaign"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a3170 @name="utm_campaign", @mode=:nullable>, "utm_term"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a2770 @name="utm_term", @mode=:nullable>, "utm_content"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a2180 @name="utm_content", @mode=:nullable>, "c_host"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a1988 @name="c_host", @mode=:nullable>, "c_script_type"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a1618 @name="c_script_type", @mode=:nullable>, "c_slot1"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a1258 @name="c_slot1", @mode=:nullable>, "c_slot2"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17a0100 @name="c_slot2", @mode=:nullable>, "c_slot3"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17afc18 @name="c_slot3", @mode=:nullable>, "c_slot4"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17af790 @name="c_slot4", @mode=:nullable>, "requestId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17aeea8 @name="requestId", @mode=:nullable>, "meta"=>#<Fluent::BigQuery::RecordSchema:0x00007f68a17add28 @name="meta", @mode=:nullable, @fields={"networkId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17ad148 @name="networkId", @mode=:nullable>, "buyerId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17acd10 @name="buyerId", @mode=:nullable>, "brandId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a17ac270 @name="brandId", @mode=:nullable>, "brandName"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20d1fa8 @name="brandName", @mode=:nullable>, "clickUrl"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20d9f50 @name="clickUrl", @mode=:nullable>, "advertiserId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20dc368 @name="advertiserId", @mode=:nullable>, "networkName"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20e2ab0 @name="networkName", @mode=:nullable>, "agencyId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20e1020 @name="agencyId", @mode=:nullable>, "agencyName"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20e7538 @name="agencyName", @mode=:nullable>, "advertiserName"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20e6f98 @name="advertiserName", @mode=:nullable>,

 "advertiserDomains"=>#<Fluent::BigQuery::RecordSchema:0x00007f68a20e5da0 @name="advertiserDomains", 
 @mode=:repeated, @fields={
 "domain"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a20e52b0 @name="domain", @mode=:nullable>}>,

  "primaryCatId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a2103170 @name="primaryCatId", @mode=:nullable>, "secondaryCatIds"=>#<Fluent::BigQuery::RecordSchema:0x00007f68a2102bd0 @name="secondaryCatIds", @mode=:repeated, @fields={"catId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a2100ce0 @name="catId", @mode=:nullable>}>, "MediaType"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a2100268 @name="MediaType", @mode=:nullable>}>, "netRevenue"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a2107b80 @name="netRevenue", @mode=:nullable>, "originalCpm"=>#<Fluent::BigQuery::FloatFieldSchema:0x00007f68a2106be0 @name="originalCpm", @mode=:nullable>, "creativeId"=>#<Fluent::BigQuery::StringFieldSchema:0x00007f68a2106550 @name="creativeId", @mode=:nullable>}>