actions-on-google / actions-on-google-nodejs Goto Github PK

We were experiencing an issue where Tax was not getting displayed in Order Summary. In troubleshooting, found for example, if units is 0, we're throwing an error 'Invalid units', and not setting the correct value -

Example for tax amount:
amount: { currencyCode: 'USD', units: 0, nanos: 680000000 }

problem is occurring in this specific code block (see highlighted portion):

Hi Team,

On the API.api I am getting

Sorry, I didn't get any response.

When I trigger app.askForSignIn(). in principle, this should tell the agent to go to the signing process.

Please have a look.

The isRequestFromAssistant jsdoc says it returns a Promise resolving with ID token if request is from a valid source. Here's what I see in the code:

  googleAuthClient.verifyIdToken(jwtToken, projectId, (err, idToken) => {
    ...
          resolve(idToken);

Here's the actual signature of verifyIdToken from https://github.com/google/google-auth-library-nodejs

public verifyIdToken(
    idToken: string, audience: string\|string[],
    callback: (err: Error, login?: LoginTicket) => void) {

As you can see, what google-auth-library considers to be the ID token is the first parameter, or jwtToken in this case. The second callback parameter called idToken here should probably be renamed to login to avoid confusion. The code above should be changed to resolve(jwtToken); to match the jsdoc.

In 1.0.4 the method ask doesn't take expected intents anymore but it still sneaks in the assistant.intent.action.TEXT as expected. Now all my intents come back as raw text. This worked in 1.0.2.

Everything stopped working.

Using express.js if I create an handler like this

const express = require("express");
const App = require("actions-on-google").ApiAiApp;
const router = express.Router();
const CULTURE_US = 'en-US';

router.post('/en-US', (request, response) => {
    function handleActions(appApi) {
        processApiAiRequest(appAi, CULTURE_US);
    }
    console.log('Request body', request.body);
    const appAi = new App({ request, response });
    appAi.handleRequest(handleActions);
});

function processApiAiRequest(appAi, culture) {
    let intent = appAi.getIntent();
    let paramValue = appAi.getArgument('fabric');
    let surfaceCapability = appAi.hasSurfaceCapability(appAi.SurfaceCapabilities.SCREEN_OUTPUT) ? 'yes' : 'no';
    appAi.ask('culture: ' + culture + 'intent: ' + intent + ' params:' + paramValue + ' screen capability: ' + surfaceCapability);
}

module.exports = router; 

the code works perfectly, but if I change the code like this it does not:

const express = require("express");
const App = require("actions-on-google").ApiAiApp;
const router = express.Router();
const CULTURE_US = 'en-US';

router.post('/it-it', (req, res) => {
    function handleActions(appApi) {
        processApiAiRequest(appAi, CULTURE_IT);
    }
    console.log('Request body', req.body);
    const appAi = new App({ req, res });
    appAi.handleRequest(handleActions);
});

function processApiAiRequest(appAi, culture) {
    let intent = appAi.getIntent();
    let paramValue = appAi.getArgument('fabric');
    let surfaceCapability = appAi.hasSurfaceCapability(appAi.SurfaceCapabilities.SCREEN_OUTPUT) ? 'yes' : 'no';
    appAi.ask('culture: ' + culture + 'intent: ' + intent + ' params:' + paramValue + ' screen capability: ' + surfaceCapability);
}

module.exports = router; 

Basically if I name the variables "req" and "res" instead of "request" and "response" I got following error:

2017-06-23T14:03:20.330203+00:00 app[web.1]: TypeError: Cannot read property 'result' of undefined
2017-06-23T14:03:20.330208+00:00 app[web.1]:     at ApiAiApp.handleRequest (/app/node_modules/actions-on-google/assistant-app.js:489:10)
2017-06-23T14:03:20.330207+00:00 app[web.1]:     at ApiAiApp.extractData_ (/app/node_modules/actions-on-google/api-ai-app.js:1405:19)
2017-06-23T14:03:20.330209+00:00 app[web.1]:     at router.post (/app/routes/webhooks.js:16:9)
2017-06-23T14:03:20.330210+00:00 app[web.1]:     at next (/app/node_modules/express/lib/router/route.js:137:13)
2017-06-23T14:03:20.330209+00:00 app[web.1]:     at Layer.handle [as handle_request] (/app/node_modules/express/lib/router/layer.js:95:5)
2017-06-23T14:03:20.330212+00:00 app[web.1]:     at /app/node_modules/express/lib/router/index.js:281:22
2017-06-23T14:03:20.330211+00:00 app[web.1]:     at Route.dispatch (/app/node_modules/express/lib/router/route.js:112:3)
2017-06-23T14:03:20.330212+00:00 app[web.1]:     at Layer.handle [as handle_request] (/app/node_modules/express/lib/router/layer.js:95:5)
2017-06-23T14:03:20.330213+00:00 app[web.1]:     at Function.process_params (/app/node_modules/express/lib/router/index.js:335:12)
2017-06-23T14:03:20.330214+00:00 app[web.1]:     at next (/app/node_modules/express/lib/router/index.js:275:10)

With the others handlers that do not use this module everything works both with "req" or "request"

the index.js is like this:

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const express = require("express");
const bodyParser = require("body-parser");
const index = require("./routes/index");
const webHooks = require("./routes/webhooks"); //the handlers with the problem
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.use('/', index);
app.use('/webhooks', webHooks);

var port = process.env.PORT || 3000;
app.listen(port, function () {
    console.log('Listening on port ' + port);
}); 

The logic in ApiAiAssistant to check whether the user object exists in the incoming request is missing parentheses, breaking it in the case that body_.originalRequest exists, but not data or data.user.

The patch below fixes it. I can make a pull request / whatever, but it seems easier if someone were to just fix this directly.

diff --git a/actions-on-google.js b/actions-on-google.js
index 01b9c30..e0d2f98 100644
--- a/actions-on-google.js
+++ b/actions-on-google.js
@@ -1492,9 +1492,9 @@ ApiAiAssistant.prototype = new Assistant();
 ApiAiAssistant.prototype.getUser = function () {
   debug('getUser');
   let self = this;
-  if (!self.body_.originalRequest &&
-      self.body_.originalRequest.data &&
-      self.body_.originalRequest.data.user) {
+    if (!(self.body_.originalRequest &&
+         self.body_.originalRequest.data &&
+         self.body_.originalRequest.data.user)) {
     self.handleError_('No user object');
     return null;
   }

I have been receiving the following error using node-js module actions-on-google v1.0.8.

/app/node_modules/actions-on-google/api-ai-assistant.js:280 
return this.body_.originalRequest.data.inputs[0].arguments[0][argName]; 
                                                              ^ 
TypeError: Cannot read property 'raw_text' of undefined 
at getArgument (/app/node_modules/actions-on-google/api-ai-assistant.js:280:68) 

It looks like the JSON can be returned from API.ai without any arguments in the input. This is then not handled by the assistant.getArgument() method call.

I would expect that in the event that the argument was not present, the method would return undefined rather than failing.

Thanks.

Looks like the ActionsSdkAssistant constructor expects request and response objects from Express rather than Node's built-in ClientRequest and ServerResponse.

Might be nice to document that.

Hi, I got this error message ("expected_inputs[0].possible_intents[0]: intent 'actions.intent.OPTION' is only supported for version 2 and above.") when I used askWithList() in ActionsSdkApp. What should I do?

2017-08-03T15:57:24.684034+00:00 app[web.1]: > [email protected] start /app
2017-08-03T15:57:24.684034+00:00 app[web.1]: > node index.js
2017-08-03T15:57:24.684035+00:00 app[web.1]:
2017-08-03T15:57:24.952550+00:00 app[web.1]: Server listening
2017-08-03T15:57:25.891946+00:00 heroku[web.1]: State changed from starting to up
2017-08-03T15:58:04.247635+00:00 heroku[router]: at=info method=POST path="/hook" host=telusassistant.herokuapp.com request_id=6c256fe8-9eb1-4d31-9951-4f630b440a60 fwd="104.197.13.204" dyno=web.1 connect=0ms service=46ms status=400 bytes=296 protocol=https
2017-08-03T15:58:04.229272+00:00 app[web.1]: hook request
2017-08-03T15:58:04.229987+00:00 app[web.1]: Request headers: {"host":"telusassistant.herokuapp.com","connection":"close","accept":"/","content-type":"application/json; charset=UTF-8","user-agent":"Apache-HttpClient/4.5.2 (Java/1.8.0_131)","accept-encoding":"gzip,deflate","x-request-id":"6c256fe8-9eb1-4d31-9951-4f630b440a60","x-forwarded-for":"104.197.13.204","x-forwarded-proto":"https","x-forwarded-port":"443","via":"1.1 vegur","connect-time":"0","x-request-start":"1501775884200","total-route-time":"0","content-length":"590"}
2017-08-03T15:58:04.229989+00:00 app[web.1]:
2017-08-03T15:58:04.229989+00:00 app[web.1]:
2017-08-03T15:58:04.230077+00:00 app[web.1]: Request body: {"id":"918a62b3-c249-4c61-b9c6-25846f75f2af","timestamp":"2017-08-03T15:58:04.009Z","lang":"en","result":{"source":"agent","resolvedQuery":"channel down","speech":"","action":"input.cmd-channel","actionIncomplete":false,"parameters":{"sys-direction":"Down"},"contexts":[],"metadata":{"intentId":"1076649c-8560-4cc0-9568-62088cc9812a","webhookUsed":"true","webhookForSlotFillingUsed":"true","intentName":"cmd-channel"},"fulfillment":{"speech":"","messages":[{"type":0,"speech":""}]},"score":1},"status":{"code":200,"errorType":"success"},"sessionId":"1de54bbe-ca94-4c66-8595-004f4ef918c1"}
2017-08-03T15:58:04.230079+00:00 app[web.1]:
2017-08-03T15:58:04.230080+00:00 app[web.1]:
2017-08-03T15:58:04.235669+00:00 app[web.1]: Can't process request TypeError: Cannot read property 'result' of undefined
2017-08-03T15:58:04.235671+00:00 app[web.1]: at ApiAiApp.extractData_ (/app/node_modules/actions-on-google/api-ai-app.js:1051:19)
2017-08-03T15:58:04.235672+00:00 app[web.1]: at ApiAiApp.handleRequest (/app/node_modules/actions-on-google/assistant-app.js:499:10)
2017-08-03T15:58:04.235673+00:00 app[web.1]: at /app/index.js:59:13
2017-08-03T15:58:04.235673+00:00 app[web.1]: at Layer.handle [as handle_request] (/app/node_modules/express/lib/router/layer.js:95:5)
2017-08-03T15:58:04.235674+00:00 app[web.1]: at next (/app/node_modules/express/lib/router/route.js:137:13)
2017-08-03T15:58:04.235674+00:00 app[web.1]: at Route.dispatch (/app/node_modules/express/lib/router/route.js:112:3)
2017-08-03T15:58:04.235675+00:00 app[web.1]: at Layer.handle [as handle_request] (/app/node_modules/express/lib/router/layer.js:95:5)
2017-08-03T15:58:04.235676+00:00 app[web.1]: at /app/node_modules/express/lib/router/index.js:281:22
2017-08-03T15:58:04.235677+00:00 app[web.1]: at Function.process_params (/app/node_modules/express/lib/router/index.js:335:12)
2017-08-03T15:58:04.235677+00:00 app[web.1]: at next (/app/node_modules/express/lib/router/index.js:275:10)

The following regex in isSsml_

return /^<speak\b[^>]*>(.*?)<\/speak>$/gi.test(text);

considers this valid:
<speak>Some text '<break time="500ms"/> Some more Text</speak>

but a multi-line SSML invalid

<speak>
Some text '<break time="500ms"/> Some more Text
</speak>

Just did a new build on Heroku, which would have pulled the latest library version and got this build failure.

/app/node_modules/actions-on-google/request-extractor.js:22
const { transformToSnakeCase } = require('./utils/transform');
^
SyntaxError: Unexpected token {
at exports.runInThisContext (vm.js:53:16)
at Module._compile (module.js:387:25)
at Object.Module._extensions..js (module.js:422:10)
at Module.load (module.js:357:32)
at Function.Module._load (module.js:314:12)
at Module.require (module.js:367:17)
at require (internal/module.js:16:19)
at Object. (/app/node_modules/actions-on-google/assistant-app.js:24:26)
at Module._compile (module.js:413:34)
at Object.Module._extensions..js (module.js:422:10)

I've rolled back to 1.1.0 for now and that's building fine.

Is it possible to send cards to users devices when using the SDK? It looks like I can do it within API.AI itself, but can't find any references in the SDK docs.

The current constructors require a request and response object, however lambda handlers only have the event/request body.

We tested a fix for it - change in the function buildResponse_
We tested it on real device for google home and it works correctly
Any reason not to enable this flow?

The fixed code in buildResponse_ function
The result was getting both responses and after no input getting the prompt for no input
We want to send card and no input promt

response.data = isStringResponse ? {
google: {

    expectUserResponse: expectUserResponse,
    isSsml: this.isSsml_(textToSpeech),
    noInputPrompts: noInputs
  }
} : {
  google: {
    expectUserResponse: expectUserResponse,
    richResponse: textToSpeech,
    noInputPrompts: noInputs  // added by zuznow 
  }
};

From 6e44302

Is type required in the payloads? This broke some of my tests in an app upgrading from 1.0.5 to 1.0.6. Also if you send an app some garbage it's just going to respond with "Cannot read property 'type' of undefined" instead of "Action Error: Missing inputs from request body", so it feels like this needs to check for presence of conversation.type or produce a better error.

I am happy to PR either.

Here's the buildLineItem implementation:

  buildLineItem (name, id) {
    return new LineItem(name, id);
  }

Here's the LineItem constructor:

  constructor (lineItemId, name) {

As you can see, buildLineItem is constructing the LineItem object with the parameters in the wrong order.

OrderAction in transactions.js is missing a customer_service as an enum (very last one): https://developers.google.com/actions/reference/rest/Shared.Types/ActionType?hl=vi

Currently when handleRequest is called, and the map of request handlers contains functions that return promises, the promise is swallowed by handleRequest and we are unable to handle the then or catch events.

An example of when this would be useful is when we have a callback function to call in our nodejs environment that we would like to call after handleRequest has internally resolved the promises.

Our solution is to return the promise from within handleRequest and invokeIntentHandler_, or true/false for non promise handlers.

We have this implemented here: sdolier@b513722

If this change is acceptable we can submit a pull request.

I'm not sure if this is an issue (at least not here) but here it goes. I'm using the API.AI assistant, everything basic stuff works. Then I make a permission request. Google grants it, I see the JSON going back to API.AI (I'm testing with the Web Simulator), but nothing gets back to my webhook. I was wondering maybe I'm doing something wrong when asking for the permission.

How to set this up with API.AI? How did you test it? Do I need an event (GOOGLE_ASSISTANT_PERMISSON?) on an intent, something similar to the welcome event mechanism?

I suspect an issue with node version as apigee is still running a very old node version. Can you please confirm which versions are compatible with the module?
Here is the error returned when I run the module on apigee. Note that it works fine on my computer.
status code: 500

{
"fault": {
"faultstring": "Script node executed prematurely: missing ; before statement\nmissing ; before statement\n at module.js:439\n at module.js:474\n at module.js:356\n at module.js:312\n at module.js:364\n at require (module.js:380)\n at /organization/environment/api/node_modules/actions-on-google/actions-on-google.js:24\n at module.js:456\n at module.js:474\n at module.js:356\n at module.js:312\n at module.js:364\n at require (module.js:380)\n at /organization/environment/api/app.js:2\n at module.js:456\n at module.js:474\n at module.js:356\n at module.js:312\n at module.js:364\n at require (module.js:380)\n at /organization/environment/api/routes/routes.js:4\n at module.js:456\n at module.js:474\n at module.js:356\n at module.js:312\n at module.js:364\n at require (module.js:380)\n at /organization/environment/api/server.js:13\n at module.js:456\n at module.js:474\n at module.js:356\n at module.js:312\n at module.js:497\n at startup (trireme.js:142)\n at trireme.js:923\n",
"detail": {
"errorcode": "scripts.node.runtime.ScriptExitedError"
}
}
}

Same as in #11, but in ActionsSdkAssistant.

Hi there

I was fighting this problem for a while, wish it was actually described in the docs but i had realized that my request object didn't have a few properties.

Any chance you might specify which properties you need to be passed to constructor ?

I would prefer if express wasn't involved hence making lightweight micro service using api ai.

thanks

I don't know if this is a bug or not, but I'm not capable of resolving it:

ActionsSdkAssistant.prototype.ask = function (inputPrompt, possibleIntents, dialogState)

if dialogState is an empty object - ask fails with an "Invalid dialog state". But there is no possibility to reset the dialogState. The underlying issue might be either that the dialogState "survives" a crash or the missing possibility to reset the state.

1. Is there a step by step tutorial for this library? Specifically, how to install and configure it on a Linux box and how to launch it under Node.js?

2. Is there a tutorial that explains how to use this node.js client library for Google Actions, a getting started or overview/tutorial? It would be extremely helpful to find one that does not use the Api.ai fulfillment side of the library since we have our own natural language processing library.

I'm working on an application that needs data to be persistent across intents (i.e., data calculated in one intent and then that data used in the next intent). How would I go about doing this? (I'm using API.ai).

question is also asked on stackoverflow as well: https://stackoverflow.com/questions/46368359/how-to-get-session-id-on-webhook

I'm making assistant app for google home and mobile assistant

I'm using action-on-google library on webhook which is recommended and handy

in my specific case I want to make userEntity from webhook which requires Session Id but I am unable to get the sessionid on webhook

according to api.ai document it sends Json to webhook like this:

{
                    "lang": "en",
                    "status": {
                        "errorType": "success",
                        "code": 200
                    },
                    "timestamp": "2017-02-09T16:06:01.908Z",
                    "sessionId": "1486656220806" "result": {
                        "parameters": {
                            "city": "Rome",
                            "name": "Ana"
                        },
                        "contexts": [],
                        "resolvedQuery": "my name is Ana and I live in Rome",
                        "source": "agent",
                        "score": 1.0,
                        "speech": "",
                        "fulfillment": {
                            "messages": [
                                {
                                    "speech": "Hi Ana! Nice to meet you!",
                                    "type": 0
                                }
                            ],
                            "speech": "Hi Ana! Nice to meet you!"
                        },
                        "actionIncomplete": false,
                        "action": "greetings",
                        "metadata": {
                            "intentId": "9f41ef7c-82fa-42a7-9a30-49a93e2c14d0",
                            "webhookForSlotFillingUsed": "false",
                            "intentName": "greetings",
                            "webhookUsed": "true"
                        }
                    },
                    "id": "ab30d214-f4bb-4cdd-ae36-31caac7a6693",
                    "originalRequest": {
                        "source": "google",
                        "data": {
                            "inputs": [
                                {
                                    "raw_inputs": [
                                        {
                                            "query": "my name is Ana and I live in Rome",
                                            "input_type": 2
                                        }
                                    ],
                                    "intent": "assistant.intent.action.TEXT",
                                    "arguments": [
                                        {
                                            "text_value": "my name is Ana and I live in Rome",
                                            "raw_text": "my name is Ana and I live in Rome",
                                            "name": "text"
                                        }
                                    ]
                                }
                            ],
                            "user": {
                                "user_id": "PuQndWs1OMjUYwVJMYqwJv0/KT8satJHAUQGiGPDQ7A="
                            },
                            "conversation": {
                                "conversation_id": "1486656220806",
                                "type": 2,
                                "conversation_token": "[]"
                            }
                        }
                    }
                }

and ofcourse it is sending it correctly but on webhook we handover the request object to action-on-google and it returns an object with a bunch of methods like ask, askWithCarousel, askWithList and etc (documented here)

the problem is that there is not method to get conversation id documented
then how do i get that session id:

my source code for reference:

/index.ts

import * as admin from 'firebase-admin';
import * as functions from 'firebase-functions';
import { Request, Response } from "express"; //interfaces
const ActionsSdkApp = require('actions-on-google').ApiAiAssistant;

import db from '../db';

// API.AI Action names
import {
inputWelcome
} from './actions'

const WELCOME_INTENT = 'input.welcome';

export const webhook = functions.https.onRequest(async (request: Request, response: Response) => {

console.log("request.body: ", request.body);
console.log("request.body.sessionId: ", request.body.sessionId);

const app = new ActionsSdkApp({ request: request, response: response });

let actionMap = new Map();
actionMap.set(WELCOME_INTENT, inputWelcome);
app.handleRequest(actionMap);
})//end of webhook http trigger

/actions/index.ts

import * as request from 'request';

export function inputWelcome(app: any) {

//I WANT SESSION ID HERE

console.log("app.conversation(): ", app.conversation());
console.log("app.AppRequest: ", app.AppRequest);
console.log("app.AppRequest.conversation: ", app.AppRequest.conversation);

console.log("app.AppRequest(): ", app.AppRequest());
console.log("app.AppRequest().conversation: ", app.AppRequest().conversation);


console.log("app.getUser().accessToken;: ", app.getUser().accessToken)
const accessToken = app.getUser().accessToken;

// MAKE USER ENTITY WITH THESE DATA:

    //     "sessionId": "current conversation id here",
    //     "name":"city",
    //      "extend":false,
    //     "entries": [
    //         {
    //             "name": "option1",
    //             "entries": [
    //                 {
    //                     "value": "option1",
    //                     "synonyms": [
    //                         "first",
    //                         "option one"
    //                     ]
    //                 }
    //             ]
    //         },
    //         {
    //             "name": "option2",
    //             "entries": [
    //                 {
    //                     "value": "option2",
    //                     "synonyms": [
    //                         "second one",
    //                         "second option"
    //                     ]
    //                 }
    //             ]
    //         },
    //         {
    //             "name": "option3",
    //             "entries": [
    //                 {
    //                     "value": "option3",
    //                     "synonyms": [
    //                         "third one",
    //                         "third option"
    //                     ]
    //                 }
    //             ]
    //         },
    //         {
    //             "name": "help",
    //             "entries": [
    //                 {
    //                     "value": "help",
    //                     "synonyms": [
    //                         "help",
    //                         "need help",
    //                         "ditn't get",
    //                         "need support"
    //                     ]
    //                 }
    //             ]
    //         }
    //     ]
    // }

// AND THEN ASK THE USER WITH SUGGESTION CHIPS

app.ask(app.buildRichResponse()
        .addSimpleResponse({
            speech: 'Hi you can start with these things',
            displayText: 'Hi you can start with these things'
        })
        .addSuggestions(['option1', 'option2', 'option3', 'Help'])
    )

Hi,

I have attached a web-hook to my actions app. Whenever i was testing the app from actual google home and simulator "web-hook" code was crashing

undefined:1
undefined
^

SyntaxError: Unexpected token u in JSON at position 0
at JSON.parse ()
at ApiAiApp.AssistantApp (/data/app/node_modules/actions-on-google/assistant-app.js:128:23)
at ApiAiApp (/data/app/node_modules/actions-on-google/api-ai-app.js:78:5)
at Server. (/data/app/app.js:16:13)
at Server. (/data/app/node_modules/engine.io/lib/server.js:472:22)
at Server. (/data/app/node_modules/socket.io/lib/index.js:307:16)
at emitTwo (events.js:125:13)
at Server.emit (events.js:213:7)
at parserOnIncoming (_http_server.js:602:12)
at HTTPParser.parserOnHeadersComplete (_http_common.js:116:23)

Later i tried printing body of the request but it was printing undefined.
Why google is not sending data to web-hook?

Can you please help? Thanks in advance

Hello,

I'm using dialogflow-app v1.5

When doing a tell:

I expect the reponse to match dialog flow expected reponse and add a displayText and a speech:
from: https://dialogflow.com/docs/fulfillment#response

Body:
{
"speech": "Barack Hussein Obama II was the 44th and current President of the United States.",
"displayText": "Barack Hussein Obama II was the 44th and current President of the United States, and the first African American to hold the office. Born in Honolulu, Hawaii, Obama is a graduate of Columbia University   and Harvard Law School, where ",
"data": {...},
"contextOut": [...],
"source": "DuckDuckGo"
}

but when i do:

assistant.tell({speech: 'Great see you at your appointment!',
             displayText: 'Great, we will see you on '});

displayText is not set in the reponse:

{
   "speech":"Great see you at your appointment!",
   "contextOut":[
      {
         "name":"downloadslist",
         "lifespan":5,
         "parameters":{         }
      }
   ],
   "data":{
      "google":{
         "expect_user_response":false,
         "rich_response":{
            "items":[
               {
                  "simple_response":{
                     "text_to_speech":"Great see you at your appointment!",
                     "display_text":"Great, we will see you on "
                  }
               }
            ],
            "suggestions":[            ]
         },
         "no_input_prompts":[         ]
      }
   }
}

Hi,

The webhook for my actions app works fine but the app does not close (ie the mic is still open for user response) when I call the tell() method on the ApiAiApp class. I tested on the Google Assistant on my Android phone and the Actions Simulator

/** 
 * @param {ApiAiApp} aiApp The ApiAi app instance
 * @param {String} speech the response to the user
 * @param {Array} suggestions a String array of suggestions
 * @param {Boolean} last true if this is the last response
 */
function buildResponse(aiApp, speech, suggestions, last){
    if (aiApp.hasSurfaceCapability(aiApp.SurfaceCapabilities.SCREEN_OUTPUT)){
        aiApp.ask(aiApp.buildRichResponse()
            .addSimpleResponse(speech)
            .addSuggestions(suggestions));
    } else {
        (last) ?  aiApp.tell(`${speech}`) : aiApp.ask(`${speech}`, []);
    }
}

Some inspection led me to use
aiApp.buildResponse_(speech, false); which works

I may be doing something wrong, I don't know. Any help is appreciated. Thanks

please refer me to document

by followupEvent i'm refering to this:
https://api.ai/docs/fulfillment#response
https://api.ai/docs/events#invoking_event_from_webhook

I have read rest of the https://dialogflow.com/ document more then twice(since it is updating frequently) and I know well how to send followup events with simple webhook (with simple rest service webhook without action-on-google),
here is my repo with the sample of it (see commented code from line 35 to 54) https://github.com/malikasinger1/quizbot-reactjs-api.ai-firebase-function/blob/master/functions/src/lib/apiai-functions.ts

but my question is how to send it with action-on-google nodejs library (https://www.npmjs.com/package/actions-on-google) since we handover the request object to action-on-google and it gives us an object app, we have methods in the object like app.ask() app.tell(), app.setContext() so i was expecting for some method like:

app.followupEvent("<event_name>",{
         "<parameter_name>":"<parameter_value>>"
      }) 
app.ask();

OR

app.askWithFollowupEvent({...
})

so what you say about this? if you don't have these method you should add one,
and please let me know some place where you guise discuss future plannings

I am not using Express.js and code fails in assistant.js request_.get (line 241)

Also I can not find version number (maybe its still in the HTTP header and I've just not found it?)

code:
if (this.request_.get(CONVERSATION_API_VERSION_HEADER)) {
  this.apiVersion_ = this.request_.get(CONVERSATION_API_VERSION_HEADER);
  debug('Assistant API version: ' + this.apiVersion_);
}
my work around is to make a dummy req.get function before "new Assistant({request: req, response: res})"

code:
req.get=(function (strName){
   if (strName=="Google-Assistant-API-Version") return "V1.0"; // Need real version
   //any others req.get functions here
   return "";
});

I have also had to do more work around's in the "Utility method to send an HTTP response" line 674
"this.response_.append","this.response_.status" and "send(response)"
in order to keep assistant.js un-touched.

Ideally assistant.js should have an option to export its HTTP responses, so that we are not locked into fixed Node server types, using something like "req.http_callback=users HTTP response function" before "new Assistant({request: req, response: res})"

This would then work for what ever your using and third party Node servers.

Please consider using git tag to tag releases.

1. console.log(app.getUserId);
   It returns:
   Here userId is the Epoch time instead of the actual userID.

Response:

{ locale: 'en-US',
  userId: '1502723585291',
  user_id: '1502723585291',
  access_token: undefined,
  userName: null 
}

My actual userId = 112695425590278732409

Expected Response:

{ locale: 'en-US',
  userId: '112695425590278732409',
  user_id: '112695425590278732409',
  access_token: undefined,
  userName: null 
}

I have a service that returns a strings with SSML strings in them. If I pass this to the ApiAiAssistant SDK wrapped in a <speak> tag, Google Home complete ignores everything inside the tag. Is this because the app hasn't been fully deployed?

Examples:

const phonetic_name = 'Product <say-as interpret-as="characters">ABC1</say-as> - Bottle';

//  Google Home says: "We found Product less-than say as interpret as equals 
//  characters greater-than A-B-C-1 less-than say as greater-than Bottle"
assistant.ask(`We found ${phonetic_name}`);

//  Google Home says: "We found less-than speak greater-than Product 
//  less-than say as interpret as equals characters greater-than A-B-C-1 less-than
//  say as greater-than Bottle less-than speak greater-than
assistant.ask(`We found <speak>${phonetic_name}</speak>`)

//  Google Home says: "We found Product Bottle"
//  Web simulator response:
//  { "response": "We found Product<unknown word to simulator>- Bottle\n" }
//
//  Further down in the debugInfo:
//  "final_response": {
//    "rich_response": {
//      "items": [{
//        "simple_response": {
//          "ssml": "<speak>We found Product <say-as interpret-as=\"characters\">ABC1</say-as> - Bottle</speak>"
//        }
//      }]
//    }
//  }
assistant.ask(`<speak>We found ${phonetic_name}</speak>`)

The last one seems to be the correct way to do this, however in both the simulator and on the Google Home device it's refusing say the things inside the say-as tags. Am I doing something wrong?

I couldn't find anything about async operation support. Can this work?

Try to use https://github.com/manekinekko/google-actions-starter and ideally would like to return a Promise from an intent handler.

Hi!
First off, great job with this package! I really like the simplicity of this package and it really has amazed me/continues to amaze me with regards to what it can do.
That said, I've frequently found it useful to get user email. Is there a way to do so without Account Linking? It doesn't make too much sense that we get to see the user's location (given access by the user), but that we aren't able to access the user's email in the same way.
Thanks.

TLDR: It'd be great if there was a way to get emails similar to the way we currently get the user's location.

The current implementation of Assistant.isSsml_() checks if the text begins with exactly '<speak>' and ends with exactly '</speak>'. This breaks down if your speak tag has any attributes, i.e. '<speak version="1.0">Hello world!</speak>'.

An improved implementation might only check for '<speak' at the beginning, or use a regular expression.

Intent name is underneath metadata.intentName, this method name is misleading.

I'm making assistant app for google home and Android mobile assistant

I'm using action-on-google library on webhook which is recommended and handy

in my specific case I want to make userEntity from webhook which requires Session Id but I am unable to get the sessionid on webhook

according to api.ai document it sends Json to webhook like this:

{
    "lang": "en", 
    "status": {
        "errorType": "success", 
        "code": 200
    }, 
    "timestamp": "2017-02-09T16:06:01.908Z", 
    "sessionId": "1486656220806"              <<<<<<<<======here is session id
    "result": {
        "parameters": {
            "city": "Rome", 
            "name": "Ana"
        }, 
        "contexts": [], 
        "resolvedQuery": "my name is Ana and I live in Rome", 
        "source": "agent", 
        "score": 1.0, 
        "speech": "", 
        "fulfillment": {
            "messages": [
                {
                    "speech": "Hi Ana! Nice to meet you!", 
                    "type": 0
                }
            ], 
            "speech": "Hi Ana! Nice to meet you!"
        }, 
        "actionIncomplete": false, 
        "action": "greetings", 
        "metadata": {
            "intentId": "9f41ef7c-82fa-42a7-9a30-49a93e2c14d0", 
            "webhookForSlotFillingUsed": "false", 
            "intentName": "greetings", 
            "webhookUsed": "true"
        }
    }, 
    "id": "ab30d214-f4bb-4cdd-ae36-31caac7a6693", 
    "originalRequest": {
        "source": "google", 
        "data": {
            "inputs": [
                {
                    "raw_inputs": [
                        {
                            "query": "my name is Ana and I live in Rome", 
                            "input_type": 2
                        }
                    ], 
                    "intent": "assistant.intent.action.TEXT", 
                    "arguments": [
                        {
                            "text_value": "my name is Ana and I live in Rome", 
                            "raw_text": "my name is Ana and I live in Rome", 
                            "name": "text"
                        }
                    ]
                }
            ], 
            "user": {
                "user_id": "PuQndWs1OMjUYwVJMYqwJv0/KT8satJHAUQGiGPDQ7A="
            }, 
            "conversation": {
                "conversation_id": "1486656220806", 
                "type": 2, 
                "conversation_token": "[]"
            }
        }
    }
}

and ofcourse it is sending it correctly but on webhook we handover the request object to action-on-google and it returns an object with a bunch of methods like ask, askWithCarousel, askWithList and etc (documented here)

the problem is that there is not method to get conversation id documented
then how do i get that session id:

my source code for reference:

/index.ts

import * as admin from 'firebase-admin';
import * as functions from 'firebase-functions';
import { Request, Response } from "express"; //interfaces
const ActionsSdkApp = require('actions-on-google').ApiAiAssistant;

import db from '../db';

// API.AI Action names
import {
    inputWelcome
} from './actions'

const WELCOME_INTENT = 'input.welcome';

export const webhook = functions.https.onRequest(async (request: Request, response: Response) => {

    console.log("request.body: ", request.body);
    console.log("request.body.sessionId: ", request.body.sessionId);

    const app = new ActionsSdkApp({ request: request, response: response });

    let actionMap = new Map();
    actionMap.set(WELCOME_INTENT, inputWelcome);
    app.handleRequest(actionMap);
})//end of webhook http trigger

/actions/index.ts

import * as request from 'request';

export function inputWelcome(app: any) {
    
    //I WANT SESSION ID HERE

    console.log("app.conversation(): ", app.conversation());
    console.log("app.AppRequest: ", app.AppRequest);
    console.log("app.AppRequest.conversation: ", app.AppRequest.conversation);
    
    console.log("app.AppRequest(): ", app.AppRequest());
    console.log("app.AppRequest().conversation: ", app.AppRequest().conversation);


    console.log("app.getUser().accessToken;: ", app.getUser().accessToken)
    const accessToken = app.getUser().accessToken;
    
// MAKE USER ENTITY WITH THESE DATA:
//    {
//     "sessionId": "current conversation id here",
//     "entities": [
//         {
//             "name": "option1",
//             "entries": [
//                 {
//                     "value": "option1",
//                     "synonyms": [
//                         "first",
//                         "option one"
//                     ]
//                 }
//             ]
//         },
//         {
//             "name": "option2",
//             "entries": [
//                 {
//                     "value": "option2",
//                     "synonyms": [
//                         "second one",
//                         "second option"
//                     ]
//                 }
//             ]
//         },
//         {
//             "name": "option3",
//             "entries": [
//                 {
//                     "value": "option3",
//                     "synonyms": [
//                         "third one",
//                         "third option"
//                     ]
//                 }
//             ]
//         },
//         {
//             "name": "help",
//             "entries": [
//                 {
//                     "value": "help",
//                     "synonyms": [
//                         "help",
//                         "need help",
//                         "ditn't get",
//                         "need support"
//                     ]
//                 }
//             ]
//         }
//     ]
// }
//
// AND THEN ASK THE USER WITH SUGGESTION CHIPS    

    app.ask(app.buildRichResponse()
       .addSimpleResponse({
            speech: `Hi you can start with these things`,
            displayText: `Hi you can start with these things`
        })
        .addSuggestions(['option1', 'option2', 'option3', 'Help'])                        
    )
}

api.ai sends the intent name in
body_.result.metadata.intentName
but the actions-on-google/api-ai-app.js library looks for the intent in:
body_.result.action

... which causes an Action Error: Missing intent from request body error.

If I have a legit "Yes" intent, API.AI will mistake the "Yes" for the permission grant's "yes" and send a "Yes" intent to me when it sends the permission result back. But when you don't have a "Yes" intent, you can use a fallback intent and a context to send yourself a "PERMISSION" intent. Wouldn't it be simpler if we exposed the original intent (assistant.intent.action.PERMISSION in this case) or just a boolean function, that says it's a permission grant response?

Hello,
i use const Assistant = require('actions-on-google').DialogflowApp; v1.5 and i have the following issue:

with Dialogflowassistant when i do a tell without any speak tag i get speech at root of json response filled correctly:

assistant.tell({speech: 'Great see you at your appointment!',
             displayText: 'Great, we will see you on '});

{
   "speech":"Great see you at your appointment!",
   "contextOut":[
      {
         "name":"downloadslist",
         "lifespan":5,
         "parameters":{         }
      }
   ],
   "data":{
      "google":{
         "expect_user_response":false,
         "rich_response":{
            "items":[
               {
                  "simple_response":{
                     "text_to_speech":"Great see you at your appointment!",
                     "display_text":"Great, we will see you on "
                  }
               }
            ],
            "suggestions":[            ]
         },
         "no_input_prompts":[         ]
      }
   }
}

But when i had speak tag assistant no longer fill speech at root:

assistant.tell({speech: '<speak>Great see you at your appointment!</speak>',
             displayText: 'Great, we will see you on '});

{
   "contextOut":[
      {
         "name":"downloadslist",
         "lifespan":5,
         "parameters":{         }
      }
   ],
   "data":{
      "google":{
         "expect_user_response":false,
         "rich_response":{
            "items":[
               {
                  "simple_response":{
                     "ssml":"<speak>Great see you at your appointment!</speak>",
                     "display_text":"Great, we will see you on "
                  }
               }
            ],
            "suggestions":[      ]
         },
         "no_input_prompts":[    ]
      }
   }
}

i think the issue comes from the fact that if SSML you add the speech to the ssml property of the google simple response binstead of text_to_speech but to set speech at root you only look for text_to_speech:

dialogflow-app.js line 983

speech: isStringResponse ? textToSpeech
        : textToSpeech.items[0].simpleResponse.textToSpeech

Also display text is never set but i will add a seperate ticket for that

When we set the handlers in the actions map and pass it to handleRequest, we don't have an option to pass the context to the executor. See the following code for example -

let teach = new (require('./teach').teach)(admin);

let actionMap = new Map();
    
actionMap.set(ACTION_LEARN, teach.handleRequest);
actionMap.set(ACTION_PRACTICE, check.handleRequest);
actionMap.set(HANDLE_ANSWER, match.handleRequest);

app.handleRequest(actionMap);

In the above example, the handleRequest methods are called in the context of ApiApp (though the ApiApp is also passed as an argument to the method) and not in the context of, say teach or check.

Is there any way of executing the methods in their proper context?

The jsdoc for this project is good enough that I was able to run it through tsd-jsdoc to generate a starting point for Typescript type definitions. I ran across a number of inaccuracies in the jsdoc though that tripped this up.

assistant-app.js

• The constructor isn't meant to called by clients, should it be marked @private?
• ANY_TYPE_PROPERTY_ and findArgument_: based on the trailing underscore in the names, should these be @private?
• SignInStatus: the members should use jsdoc like the other enums in this file
• askForTransactionDecision: the order param type should be Order not Object
• getInputType: InputTypes changed from number to string with V2. The return type for this should probably be number|string, but definitely not number
• getDeliveryAddress: the return type references a non-existent DeliveryAddress type. Should this be Location?

dialogflow-app.js

• setContext: int isn't a valid type for the lifespan param. It should be number

response-builder.js

• SimpleResponse: change displayText to [displayText] to mark it as optional (http://usejsdoc.org/tags-property.html)
• OptionItem: the typedef is redundant, it's declared as a class further down in the file
• List and Carousel: Array<OptionItems> isn't a valid type for items, it should be Array<OptionItem>

transactions.js

• Price, TransitInfo, and FulfillmentInfo: the amount.nanos property is marked as optional, but it's using the syntax for param not property (http://usejsdoc.org/tags-property.html)
• ActionPaymentTransactionConfig: boolean isn't the right type for the type param
• TransactionDecision:
  • Lots of missing jsdoc here, especially paymentInfo, which is mostly documented at https://developers.google.com/actions/transactions/dev-guide#propose_the_order_to_the_user
  • Is string the right type for order.orderDate.seconds and should order.orderDate.nanos be optional? This isn't consistent with the timestamps in TransitInfo and FulfillmentInfo.
  • deliveryAddress references a non-existent DeliveryAddress type. Should this be Location?
• TransactionValues.PaymentType: "List of possible item types" should probably say "payment" instead
• Should reverseOrderStateInfo be @private?
• Order: the termsOfServiceUrl type should be lowercase string
• OrderUpdate: for the setUserNotification text param, text isn't a valid type

One example
ActionsSdkAssistant.prototype.ask = function (inputPrompt, possibleIntents, dialogState) {
debug('ask: inputPrompt=%s, possibleIntents=%s, dialogState=%s',
inputPrompt, possibleIntents, dialogState);

all variables are Objects, so the output is pretty meaningless if the variables are not "stringified"

Hi,

I'm getting this error when the program calls assistant.tell. Does anyone have this error when they are using action-on-google together with express?

Thanks

Looks like there are doc strings on everything in the source, but is there an html version of these anywhere?

Would you guys be willing to either create/accept via pull req a TypeScript declarations file? It would involve creating a TypeScript declaration file and then adding a 'types' reference to it in package.json.

https://www.typescriptlang.org/docs/handbook/declaration-files/introduction.html

Thanks!