Skip to content

Commit

Permalink
Fixes microsoft#1847 Clicking on a card maps to incorrect activity pr…
Browse files Browse the repository at this point in the history 
…operty

Fixes microsoft#2041 Android VA App log and manual resend GPS cord
Fixes microsoft#1973 Android app talks after typing input
Fixes microsoft#2002 Google Maps and Waze directions implemented
Ensured that plug-in apps receive all appropriate events and all unknown
events
Updated Bot Activity Value. It's now an "ActivityValue"instead of just a
plain String
  • Loading branch information
@abiemann
abiemann committed Aug 8, 2019
1 parent 29e8dfd commit 5fd3093
Show file tree
Hide file tree
Showing 20 changed files with 545 additions and 152 deletions.
2 changes: 2 additions & 0 deletions ...bot/builder/solutions/virtualassistant/activities/configuration/DefaultConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,7 @@ public class DefaultConfiguration {

public static final String LOCALE = "en-us";

public static final String KEYWORD = "computer";

// please note that the default colors are read from /res/values/colors.xml
}
1 change: 1 addition & 0 deletions solutions/android/VirtualAssistantClient/app/src/main/AndroidManifest.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
<!-- NORMAL -->
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" /> <!-- for ContextCompat.startForegroundService -->
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE"/>
<!-- DANGEROUS -->
<uses-permission android:name="android.permission.RECORD_AUDIO" /> <!-- FOR SPEECH COMMANDS -->
<uses-permission android:name="android.permission.ACCESS_FINE_LOCATION" />
Expand Down
9 changes: 7 additions & 2 deletions ...pp/src/main/aidl/com/microsoft/bot/builder/solutions/virtualassistant/ISpeechService.aidl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,20 @@ interface ISpeechService {
void sendTextMessage(String msg);
void initializeSpeechSdk(boolean haveRecordAudioPermission);
void connectAsync();
void startLocationUpdates();
void resetBot();
String getConfiguration();// the String is "Configuration" as JSON
void setConfiguration(String json);// the String is "Configuration" as JSON
void sendLocationEvent(String lat, String lon);
void requestWelcomeCard();
void injectReceivedActivity(String json);
void listenOnceAsync();
void sendActivityMessageAsync(String msg);
String getSuggestedActions();//the String is "List<CardAction>" as JSON
void clearSuggestedActions();
void startKeywordListeningAsync(String keyword);
void stopKeywordListening();
void stopAnyTTS();
void startLocationUpdates();
String getDateSentLocationEvent();
void sendLocationEvent(String lat, String lon);
void sendLocationUpdate();
}
52 changes: 47 additions & 5 deletions ...in/java/com/microsoft/bot/builder/solutions/virtualassistant/activities/BaseActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import android.view.inputmethod.InputMethodManager;
import android.widget.Toast;

import com.microsoft.bot.builder.solutions.directlinespeech.ConfigurationManager;
import com.microsoft.bot.builder.solutions.directlinespeech.model.Configuration;
import com.microsoft.bot.builder.solutions.virtualassistant.ISpeechService;
import com.microsoft.bot.builder.solutions.virtualassistant.R;

Expand All @@ -30,20 +32,22 @@
*/
public abstract class BaseActivity extends AppCompatActivity {

// Constants
// CONSTANTS
public static final String LOGTAG = "BaseActivity";
private static final Integer PERMISSION_REQUEST_RECORD_AUDIO = 101;
private static final Integer PERMISSION_REQUEST_FINE_LOCATION = 102;
public static final String SHARED_PREFS_NAME = "my_shared_prefs";
protected static final String SHARED_PREF_SHOW_TEXTINPUT = "SHARED_PREF_SHOW_TEXTINPUT";
protected static final String SHARED_PREF_SHOW_FULL_CONVERSATION = "SHARED_PREF_SHOW_FULL_CONVERSATION";
public static final String SHARED_PREF_DARK_MODE = "SHARED_PREF_DARK_MODE";
protected static final String SHARED_PREF_ENABLE_KWS = "SHARED_PREF_ENABLE_KWS";

// State
// STATE
private SharedPreferences sharedPreferences;
protected ISpeechService speechServiceBinder;
protected ConfigurationManager configurationManager;

// Override these
// OVERRIDE THESE
protected void permissionDenied(String manifestPermission){}
protected void permissionGranted(String manifestPermission){}
protected void serviceConnected(){}
Expand All @@ -53,6 +57,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
sharedPreferences = getSharedPreferences(SHARED_PREFS_NAME, MODE_PRIVATE);
setupMainWindowDisplayMode();
configurationManager = new ConfigurationManager(this);
}

@Override
Expand Down Expand Up @@ -194,7 +199,7 @@ public void doBindService() {
Intent intent = new Intent();
intent.setClassName("com.microsoft.bot.builder.solutions.virtualassistant","com.microsoft.bot.builder.solutions.virtualassistant.service.SpeechService");
boolean success = bindService(intent, myConnection, Context.BIND_AUTO_CREATE);
Log.d(LOGTAG,"success = "+success);
Log.d(LOGTAG,"doBindService() success = "+success);
}

public ServiceConnection myConnection = new ServiceConnection() {
Expand All @@ -215,7 +220,7 @@ public void onServiceDisconnected(ComponentName className) {
protected void initializeAndConnect(){
if (speechServiceBinder != null) {
try {
speechServiceBinder.initializeSpeechSdk(true);
//speechServiceBinder.initializeSpeechSdk(true);
speechServiceBinder.connectAsync();
} catch (RemoteException exception){
Log.e(LOGTAG, exception.getMessage());
Expand All @@ -225,4 +230,41 @@ protected void initializeAndConnect(){
}
}

/**
* enable or disable Keyword Sensing
* @param enable true to enable
* @return true if KWS is enabled, otherwise false
*/
protected boolean setKwsState(boolean enable) {
boolean enabledKws = false;

if (enable) {
if ((ActivityCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED)){
Log.w(LOGTAG, "Insufficient permissions to access audio. Skipping KWS toggle operation");
return false;
}

try {
final Configuration configuration = configurationManager.getConfiguration();
String keyword = configuration.keyword;
if (keyword != null) {
speechServiceBinder.startKeywordListeningAsync(keyword);
}
enabledKws = true;
}
catch (RemoteException e){
Log.e(LOGTAG, e.getMessage());
}
}
else {
try {
speechServiceBinder.stopKeywordListening();
} catch (RemoteException e) {
e.printStackTrace();
}
}

return enabledKws;
}

}
96 changes: 64 additions & 32 deletions ...va/com/microsoft/bot/builder/solutions/virtualassistant/activities/main/MainActivity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.microsoft.bot.builder.solutions.directlinespeech.ConfigurationManager;
import com.microsoft.bot.builder.solutions.directlinespeech.model.Configuration;
import com.microsoft.bot.builder.solutions.virtualassistant.R;
import com.microsoft.bot.builder.solutions.virtualassistant.activities.BaseActivity;
Expand All @@ -62,6 +61,7 @@
import butterknife.OnCheckedChanged;
import butterknife.OnClick;
import butterknife.OnEditorAction;
import butterknife.OnTextChanged;
import client.model.BotConnectorActivity;
import client.model.CardAction;
import client.model.InputHints;
Expand Down Expand Up @@ -89,6 +89,7 @@ public class MainActivity extends BaseActivity
@BindView(R.id.speech_detection) TextView detectedSpeechToText;
@BindView(R.id.mic_image) ImageView micImage;
@BindView(R.id.animated_assistant) AppCompatImageView animatedAssistant;
@BindView(R.id.switch_enable_kws) SwitchCompat switchEnableKws;

// CONSTANTS
private static final int CONTENT_VIEW = R.layout.activity_main;
Expand All @@ -103,8 +104,8 @@ public class MainActivity extends BaseActivity
private boolean launchedAsAssistant;
private Gson gson;
private SfxManager sfxManager;
private ConfigurationManager configurationManager;
private boolean willListenAgain;
private boolean enableKws;

@Override
protected void onCreate(Bundle savedInstanceState) {
Expand All @@ -114,7 +115,6 @@ protected void onCreate(Bundle savedInstanceState) {

handler = new Handler(Looper.getMainLooper());
gson = new Gson();
configurationManager = new ConfigurationManager(this);

setupChatRecyclerView();
setupSuggestedActionsRecyclerView();
Expand All @@ -125,6 +125,8 @@ protected void onCreate(Bundle savedInstanceState) {
showFullConversation = getBooleanSharedPref(SHARED_PREF_SHOW_FULL_CONVERSATION);
switchShowFullConversation.setChecked(showFullConversation);
switchNightMode.setChecked(AppCompatDelegate.getDefaultNightMode() == AppCompatDelegate.MODE_NIGHT_YES);
enableKws = getBooleanSharedPref(SHARED_PREF_ENABLE_KWS);
switchEnableKws.setChecked(enableKws);

// NAV DRAWER
ActionBarDrawerToggle toggle = new ActionBarDrawerToggle( this, drawer, R.string.navigation_drawer_open, R.string.navigation_drawer_close);
Expand Down Expand Up @@ -232,16 +234,15 @@ protected void permissionDenied(String manifestPermission) {
protected void permissionGranted(String manifestPermission) {
// this code is triggered when a user launches app a 1st time and doesn't have permisison yet
if (manifestPermission.equals(Manifest.permission.RECORD_AUDIO)){
// initializeAndConnect();
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED) {
requestFineLocationPermissions();
} else {
initializeAndConnect();
}
}
if (manifestPermission.equals(Manifest.permission.ACCESS_FINE_LOCATION)){
try {
if (speechServiceBinder != null) speechServiceBinder.startLocationUpdates();
initializeAndConnect();
//initializeAndConnect();
} catch (RemoteException exception){
Log.e(LOGTAG, exception.getMessage());
}
Expand All @@ -254,7 +255,11 @@ protected void serviceConnected() {
// this code is triggered after the service is bound.
// Binding is started in onStart(), so expect this callback to trigger after onStart()
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
initializeAndConnect();
// initializeAndConnect();
boolean enabled = setKwsState(enableKws);
if (!enabled && enableKws) {
switchEnableKws.setChecked(false);
}
}

try {
Expand Down Expand Up @@ -306,15 +311,34 @@ private void hideListeningAnimation(){
@OnClick(R.id.mic_image)
public void onClickAssistant() {
try {
speechServiceBinder.stopAnyTTS();
showListeningAnimation();
speechServiceBinder.listenOnceAsync();
} catch (RemoteException exception){
Log.e(LOGTAG, exception.getMessage());
}
}

@OnCheckedChanged(R.id.switch_enable_kws)
public void onCheckedChangedEnableKws(CompoundButton button, boolean checked){

if (speechServiceBinder != null) {
// if there's a connection to the service, go ahead and toggle Kws
enableKws = setKwsState(checked);//returns true only if Kws is turned on
} else {
// defer toggling Kws for later, for now records users' wishes
enableKws = checked;
}

putBooleanSharedPref(SHARED_PREF_ENABLE_KWS, enableKws);

if (checked && !enableKws) {
switchEnableKws.setChecked(false);
}
}

@OnCheckedChanged(R.id.switch_show_textinput)
public void OnShowTextInput(CompoundButton button, boolean checked){
public void OnCheckedChangedShowTextInput(CompoundButton button, boolean checked){
alwaysShowTextInput = checked;
putBooleanSharedPref(SHARED_PREF_SHOW_TEXTINPUT, checked);
if (alwaysShowTextInput)
Expand All @@ -324,14 +348,14 @@ public void OnShowTextInput(CompoundButton button, boolean checked){
}

@OnCheckedChanged(R.id.switch_show_full_conversation)
public void OnShowFullConversation(CompoundButton button, boolean checked){
public void OnCheckedChangedShowFullConversation(CompoundButton button, boolean checked){
showFullConversation = checked;
putBooleanSharedPref(SHARED_PREF_SHOW_FULL_CONVERSATION, checked);
chatAdapter.setShowFullConversation(showFullConversation);
}

@OnCheckedChanged(R.id.switch_night_mode)
public void OnEnableNightMode(CompoundButton button, boolean checked){
public void OnCheckedChangedEnableNightMode(CompoundButton button, boolean checked){
putBooleanSharedPref(SHARED_PREF_DARK_MODE, checked);
AppCompatDelegate.setDefaultNightMode(checked?AppCompatDelegate.MODE_NIGHT_YES:AppCompatDelegate.MODE_NIGHT_NO);
getDelegate().applyDayNight();
Expand All @@ -349,6 +373,26 @@ boolean onEditorAction(int actionId, KeyEvent key){
return handled;
}

@OnTextChanged(R.id.textinput)
protected void onTextChanged(CharSequence text) {
try {
if (speechServiceBinder != null) speechServiceBinder.stopAnyTTS();
} catch (RemoteException e) {
e.printStackTrace();
}
}

// @OnFocusChange(R.id.textinput)
// void onFocusChanged(boolean focused) {
// if (focused) {
// try {
// speechServiceBinder.stopAnyTTS();
// } catch (RemoteException e) {
// e.printStackTrace();
// }
// }
// }

// send text message
private void sendTextMessage(String msg){
if (msg == null || msg.length() == 0) return;
Expand Down Expand Up @@ -486,33 +530,21 @@ private void playMediaStream(String mediaStream) {
catch(IOException e) {
Log.e(LOGTAG, "IOexception " + e.getMessage());
}

}

// concrete implementation of ChatViewholder.OnClickListener
// concrete implementation of ViewholderBot.OnClickListener
@Override
public void adaptiveCardClick(int position, String speak) {
CardAction cardAction = null;

try {
String json = speechServiceBinder.getSuggestedActions();
List<CardAction> list = gson.fromJson(json, new TypeToken<List<CardAction>>(){}.getType());
if (list != null && list.size() > position){
cardAction = list.get(position);
public void adaptiveCardClick(int position, String clickData) {
if (clickData != null) {
try {
speechServiceBinder.stopAnyTTS();
willListenAgain = false;// no need to listen again since user clicked adaptive card
} catch (RemoteException e) {
e.printStackTrace();
}
} catch (RemoteException exception){
Log.e(LOGTAG, exception.getMessage());
}

// respond to the bot with the suggestedAction[position].Value if possible
if (cardAction != null && cardAction.getValue() != null) {
String value = (String) cardAction.getValue();
sendTextMessage(value);
} else {
sendTextMessage(speak);
sendTextMessage(clickData);
sfxManager.playEarconProcessing();
}

sfxManager.playEarconProcessing();
}

// concrete implementation of ActionsViewholder.OnClickListener
Expand Down
17 changes: 13 additions & 4 deletions ...rosoft/bot/builder/solutions/virtualassistant/activities/main/chatlist/ViewholderBot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,18 @@ void bind(@NonNull ChatModel chatModel, AppCompatActivity parentActivity, @NonNu
String cardBodyJson = cardContent.toString();
LogUtils.logLongInfoMessage(LOGTAG, "Received Card: " + cardBodyJson);// this JSON can be used with https://adaptivecards.io/designer/

// collect what the payload is when clicked
final String speak = cardContent.getString("speak");
final int position = x;
// collect payload for the click event
String selectActionData = null;
JSONObject selectAction = cardContent.getJSONObject("selectAction");
if (selectAction != null) {
String selectActionType = selectAction.getString("type");
if (selectActionType != null && selectActionType.equals("Action.Submit")) {
selectActionData = selectAction.getString("data");
}
}

final int clickPosition = x;
final String clickData = selectActionData;

ParseResult parseResult = AdaptiveCard.DeserializeFromString(cardBodyJson, AdaptiveCardRenderer.VERSION);
AdaptiveCard adaptiveCard = parseResult.GetAdaptiveCard();
Expand All @@ -91,7 +100,7 @@ void bind(@NonNull ChatModel chatModel, AppCompatActivity parentActivity, @NonNu
adaptiveCardRendered.setFocusable(false);
adaptiveCardRendered.setFocusableInTouchMode(false);
adaptiveCardRendered.setOnClickListener(v -> {
onClickListener.adaptiveCardClick(position, speak); // callback to activity
onClickListener.adaptiveCardClick(clickPosition, clickData); // callback to activity
});

// add the card to its individual container to allow for resizing
Expand Down
Loading

0 comments on commit 5fd3093

Please sign in to comment.