308 lines
14 KiB
Go
308 lines
14 KiB
Go
package device
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
"data-terminal-cloud/internal/deviceauth"
|
|
"data-terminal-cloud/internal/idempotency"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
var (
|
|
ErrTelemetryConfigAhead = errors.New("reported configuration is ahead of desired configuration")
|
|
ErrTelemetryConversationMissing = errors.New("telemetry conversation is missing or belongs to another device")
|
|
ErrTelemetryCommandConflict = errors.New("telemetry command result conflicts with command state")
|
|
)
|
|
|
|
type TelemetryNetwork struct {
|
|
Type string `json:"type"`
|
|
RSSI *int `json:"rssi,omitempty"`
|
|
}
|
|
|
|
type TelemetryRTC struct {
|
|
Joined bool `json:"joined"`
|
|
Uplink bool `json:"uplink"`
|
|
Downlink bool `json:"downlink"`
|
|
}
|
|
|
|
type TelemetryRecordingQueue struct {
|
|
PendingSegments int `json:"pending_segments"`
|
|
FailedSegments int `json:"failed_segments"`
|
|
LastFailureCode string `json:"last_failure_code,omitempty"`
|
|
LastFailureAt *time.Time `json:"last_failure_at,omitempty"`
|
|
}
|
|
|
|
type TelemetryStatus struct {
|
|
FirmwareVersion string `json:"firmware_version"`
|
|
ReportedConfigRevision int64 `json:"reported_config_revision"`
|
|
Network TelemetryNetwork `json:"network"`
|
|
RTC TelemetryRTC `json:"rtc"`
|
|
RecordingQueue TelemetryRecordingQueue `json:"recording_queue"`
|
|
ConversationStatus string `json:"conversation_status,omitempty"`
|
|
}
|
|
|
|
type TelemetryError struct {
|
|
Code string `json:"code"`
|
|
Severity string `json:"severity"`
|
|
}
|
|
|
|
type TelemetryCommandResult struct {
|
|
CommandID string `json:"command_id"`
|
|
Status string `json:"status"`
|
|
ResultSummary map[string]any `json:"result_summary"`
|
|
}
|
|
|
|
type TelemetryEvent struct {
|
|
EventKey string `json:"event_key"`
|
|
SequenceNo int64 `json:"sequence_no"`
|
|
OccurredAt time.Time `json:"occurred_at"`
|
|
EventType string `json:"event_type"`
|
|
ConversationID string `json:"conversation_id,omitempty"`
|
|
Status *TelemetryStatus `json:"status,omitempty"`
|
|
Error *TelemetryError `json:"error,omitempty"`
|
|
CommandResult *TelemetryCommandResult `json:"command_result,omitempty"`
|
|
}
|
|
|
|
type TelemetryBatchInput struct {
|
|
Events []TelemetryEvent `json:"events"`
|
|
}
|
|
|
|
type TelemetryConflict struct {
|
|
EventKey string `json:"event_key"`
|
|
SequenceNo int64 `json:"sequence_no"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
type TelemetryConflictError struct {
|
|
Conflicts []TelemetryConflict
|
|
}
|
|
|
|
func (e *TelemetryConflictError) Error() string { return "telemetry event conflicts with stored event" }
|
|
|
|
type TelemetryAction struct {
|
|
Type string `json:"type"`
|
|
ConversationID string `json:"conversation_id,omitempty"`
|
|
}
|
|
|
|
type TelemetryBatchResponse struct {
|
|
AcceptedCount int `json:"accepted_count"`
|
|
DuplicateCount int `json:"duplicate_count"`
|
|
DesiredConfigRevision int64 `json:"desired_config_revision"`
|
|
Actions []TelemetryAction `json:"actions"`
|
|
}
|
|
|
|
type TelemetryService struct {
|
|
pool *pgxpool.Pool
|
|
executor *idempotency.Executor
|
|
}
|
|
|
|
func NewTelemetryService(pool *pgxpool.Pool, executor *idempotency.Executor) (*TelemetryService, error) {
|
|
if pool == nil || executor == nil {
|
|
return nil, errors.New("telemetry database and idempotency executor are required")
|
|
}
|
|
return &TelemetryService{pool: pool, executor: executor}, nil
|
|
}
|
|
|
|
func (s *TelemetryService) Submit(ctx context.Context, identity deviceauth.Identity, idempotencyKey string, rawRequest []byte, input TelemetryBatchInput) (TelemetryBatchResponse, bool, error) {
|
|
tenantID := identity.TenantID
|
|
result, err := s.executor.Execute(ctx, idempotency.Request{
|
|
TenantID: &tenantID, ActorKey: "device:" + identity.PublicID, Scope: "POST /device/v2/telemetry:batch",
|
|
Key: idempotencyKey, RequestHash: idempotency.Hash(rawRequest), ExpiresAfter: 24 * time.Hour,
|
|
}, func(operationContext context.Context, tx pgx.Tx) (idempotency.Result, error) {
|
|
response, submitErr := s.submit(operationContext, tx, identity, input)
|
|
if submitErr != nil {
|
|
return idempotency.Result{}, submitErr
|
|
}
|
|
body, marshalErr := json.Marshal(response)
|
|
if marshalErr != nil {
|
|
return idempotency.Result{}, marshalErr
|
|
}
|
|
resourceID := identity.ID
|
|
return idempotency.Result{StatusCode: http.StatusOK, Body: body, ResourceID: &resourceID}, nil
|
|
})
|
|
if err != nil {
|
|
return TelemetryBatchResponse{}, false, err
|
|
}
|
|
var response TelemetryBatchResponse
|
|
if err := json.Unmarshal(result.Body, &response); err != nil {
|
|
return TelemetryBatchResponse{}, result.Replayed, fmt.Errorf("decode telemetry response: %w", err)
|
|
}
|
|
return response, result.Replayed, nil
|
|
}
|
|
|
|
func (s *TelemetryService) submit(ctx context.Context, tx pgx.Tx, identity deviceauth.Identity, input TelemetryBatchInput) (TelemetryBatchResponse, error) {
|
|
var desiredRevision int64
|
|
var reportedRevision *int64
|
|
if err := tx.QueryRow(ctx, `SELECT desired_revision,reported_revision FROM device_twin WHERE device_id=$1::uuid AND tenant_id=$2::uuid FOR UPDATE`, identity.ID, identity.TenantID).Scan(&desiredRevision, &reportedRevision); err != nil {
|
|
return TelemetryBatchResponse{}, fmt.Errorf("read device telemetry state: %w", err)
|
|
}
|
|
response := TelemetryBatchResponse{DesiredConfigRevision: desiredRevision, Actions: []TelemetryAction{}}
|
|
conflicts := make([]TelemetryConflict, 0)
|
|
stopActions := make(map[string]struct{})
|
|
|
|
for _, event := range input.Events {
|
|
eventPayload, err := json.Marshal(event)
|
|
if err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
digest := idempotency.Hash(eventPayload)
|
|
rows, err := tx.Query(ctx, `SELECT event_key,sequence_no,payload_digest FROM device_telemetry_event WHERE device_id=$1::uuid AND (event_key=$2 OR sequence_no=$3) FOR UPDATE`, identity.ID, event.EventKey, event.SequenceNo)
|
|
if err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
matches := 0
|
|
exact := false
|
|
for rows.Next() {
|
|
matches++
|
|
var storedKey, storedDigest string
|
|
var storedSequence int64
|
|
if err := rows.Scan(&storedKey, &storedSequence, &storedDigest); err != nil {
|
|
rows.Close()
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
if storedKey == event.EventKey && storedSequence == event.SequenceNo && storedDigest == digest {
|
|
exact = true
|
|
}
|
|
}
|
|
rows.Close()
|
|
if rows.Err() != nil {
|
|
return TelemetryBatchResponse{}, rows.Err()
|
|
}
|
|
if matches > 0 {
|
|
if matches == 1 && exact {
|
|
response.DuplicateCount++
|
|
continue
|
|
}
|
|
conflicts = append(conflicts, TelemetryConflict{EventKey: event.EventKey, SequenceNo: event.SequenceNo, Reason: "EVENT_KEY_OR_SEQUENCE_ALREADY_USED"})
|
|
continue
|
|
}
|
|
|
|
var conversationInternalID *string
|
|
if event.ConversationID != "" {
|
|
var internalID, conversationStatus string
|
|
err := tx.QueryRow(ctx, `SELECT id::text,status FROM conversation WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND device_id=$3::uuid`, event.ConversationID, identity.TenantID, identity.ID).Scan(&internalID, &conversationStatus)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return TelemetryBatchResponse{}, ErrTelemetryConversationMissing
|
|
}
|
|
if err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
conversationInternalID = &internalID
|
|
if conversationStatus == "STOPPING" || conversationStatus == "COMPLETED" || conversationStatus == "FAILED" || conversationStatus == "EXPIRED" {
|
|
stopActions[event.ConversationID] = struct{}{}
|
|
}
|
|
}
|
|
|
|
var errorCode, errorSeverity *string
|
|
var commandInternalID *string
|
|
switch event.EventType {
|
|
case "status":
|
|
if event.Status.ReportedConfigRevision > desiredRevision {
|
|
return TelemetryBatchResponse{}, ErrTelemetryConfigAhead
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE device_twin SET
|
|
telemetry_sequence=$3,
|
|
reported_revision=CASE WHEN reported_revision IS NULL OR reported_revision<=$4 THEN $4 ELSE reported_revision END,
|
|
reported_config=CASE WHEN reported_revision IS NULL OR reported_revision<=$4 THEN reported_config || jsonb_build_object('telemetry',$5::jsonb) ELSE reported_config END,
|
|
firmware_version=NULLIF($6,''),last_reported_at=now(),updated_at=now()
|
|
WHERE device_id=$1::uuid AND tenant_id=$2::uuid AND telemetry_sequence<$3`, identity.ID, identity.TenantID,
|
|
event.SequenceNo, event.Status.ReportedConfigRevision, eventPayload, event.Status.FirmwareVersion); err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
if reportedRevision == nil || *reportedRevision < event.Status.ReportedConfigRevision {
|
|
value := event.Status.ReportedConfigRevision
|
|
reportedRevision = &value
|
|
}
|
|
case "error":
|
|
errorCode, errorSeverity = &event.Error.Code, &event.Error.Severity
|
|
if _, err := tx.Exec(ctx, `UPDATE device_twin SET telemetry_sequence=$3,last_error_code=$4,last_error_severity=$5,last_error_at=$6,last_reported_at=now(),updated_at=now() WHERE device_id=$1::uuid AND tenant_id=$2::uuid AND telemetry_sequence<$3`, identity.ID, identity.TenantID, event.SequenceNo, event.Error.Code, event.Error.Severity, event.OccurredAt); err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
case "command_result":
|
|
internalID, err := applyTelemetryCommandResult(ctx, tx, identity, *event.CommandResult)
|
|
if err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
commandInternalID = &internalID
|
|
if _, err := tx.Exec(ctx, `UPDATE device_twin SET telemetry_sequence=$3,last_reported_at=now(),updated_at=now() WHERE device_id=$1::uuid AND tenant_id=$2::uuid AND telemetry_sequence<$3`, identity.ID, identity.TenantID, event.SequenceNo); err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
}
|
|
|
|
if _, err := tx.Exec(ctx, `INSERT INTO device_telemetry_event(
|
|
tenant_id,device_id,conversation_id,event_key,sequence_no,event_type,occurred_at,payload_digest,payload,error_code,error_severity,command_id
|
|
) VALUES($1::uuid,$2::uuid,$3::uuid,$4,$5,$6,$7,$8,$9::jsonb,$10,$11,$12::uuid)`, identity.TenantID, identity.ID,
|
|
conversationInternalID, event.EventKey, event.SequenceNo, event.EventType, event.OccurredAt, digest, eventPayload,
|
|
errorCode, errorSeverity, commandInternalID); err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
response.AcceptedCount++
|
|
}
|
|
if len(conflicts) > 0 {
|
|
return TelemetryBatchResponse{}, &TelemetryConflictError{Conflicts: conflicts}
|
|
}
|
|
if _, err := tx.Exec(ctx, `SELECT record_device_presence($1::uuid,$2::uuid,now())`, identity.TenantID, identity.ID); err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
if reportedRevision == nil || *reportedRevision < desiredRevision {
|
|
response.Actions = append(response.Actions, TelemetryAction{Type: "refresh_config"})
|
|
}
|
|
var hasCommands bool
|
|
if err := tx.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM device_command WHERE tenant_id=$1::uuid AND device_id=$2::uuid AND status IN ('QUEUED','DELIVERED','ACKNOWLEDGED') AND expires_at>now())`, identity.TenantID, identity.ID).Scan(&hasCommands); err != nil {
|
|
return TelemetryBatchResponse{}, err
|
|
}
|
|
if hasCommands {
|
|
response.Actions = append(response.Actions, TelemetryAction{Type: "pull_commands"})
|
|
}
|
|
for conversationID := range stopActions {
|
|
response.Actions = append(response.Actions, TelemetryAction{Type: "stop_conversation", ConversationID: conversationID})
|
|
}
|
|
return response, nil
|
|
}
|
|
|
|
func applyTelemetryCommandResult(ctx context.Context, tx pgx.Tx, identity deviceauth.Identity, input TelemetryCommandResult) (string, error) {
|
|
resultJSON, err := json.Marshal(input.ResultSummary)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
var internalID string
|
|
err = tx.QueryRow(ctx, `UPDATE device_command SET status=$4,result_summary=$5::jsonb,
|
|
acknowledged_at=COALESCE(acknowledged_at,now()),
|
|
completed_at=CASE WHEN $4 IN ('SUCCEEDED','FAILED') THEN COALESCE(completed_at,now()) ELSE completed_at END,
|
|
version=version+1,updated_at=now()
|
|
WHERE public_id=$1::uuid AND tenant_id=$2::uuid AND device_id=$3::uuid AND expires_at>now()
|
|
AND (($4='ACKNOWLEDGED' AND status='DELIVERED') OR ($4 IN ('SUCCEEDED','FAILED') AND status IN ('DELIVERED','ACKNOWLEDGED')) OR status=$4)
|
|
RETURNING id::text`, input.CommandID, identity.TenantID, identity.ID, input.Status, resultJSON).Scan(&internalID)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return "", ErrTelemetryCommandConflict
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
assignmentStatus, progress := firmwareAssignmentProgress(CommandAcknowledgement{Status: input.Status, ResultSummary: input.ResultSummary})
|
|
if assignmentStatus != "" {
|
|
if _, err := tx.Exec(ctx, `UPDATE firmware_assignment SET status=$2,progress_percent=COALESCE($3,progress_percent),last_result=$4::jsonb,last_reported_at=now(),updated_at=now() WHERE command_id=$1::uuid`, internalID, assignmentStatus, progress, resultJSON); err != nil {
|
|
return "", err
|
|
}
|
|
if err := recordFirmwareAssignmentEvent(ctx, tx, internalID, input.Status, assignmentStatus, progress, resultJSON); err != nil {
|
|
return "", err
|
|
}
|
|
if _, err := tx.Exec(ctx, `UPDATE firmware_rollout ro SET
|
|
status=CASE WHEN EXISTS(SELECT 1 FROM firmware_assignment fa WHERE fa.rollout_id=ro.id AND fa.status='FAILED') THEN 'PAUSED' ELSE 'COMPLETED' END,
|
|
completed_at=CASE WHEN NOT EXISTS(SELECT 1 FROM firmware_assignment fa WHERE fa.rollout_id=ro.id AND fa.status NOT IN ('CONFIRMED','FAILED')) THEN now() ELSE completed_at END,
|
|
updated_at=now()
|
|
WHERE id IN (SELECT fa.rollout_id FROM firmware_assignment fa WHERE fa.command_id=$1::uuid)
|
|
AND NOT EXISTS(SELECT 1 FROM firmware_assignment open WHERE open.rollout_id=ro.id AND open.status NOT IN ('CONFIRMED','FAILED'))`, internalID); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
return internalID, nil
|
|
}
|