@conference{SZ:MZaamas24,

title = {Observer-Aware Planning with Implicit and Explicit Communication},

author = {Shuwa Miura and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MZaamas24.pdf},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the The 23rd International Conference on Autonomous Agents and Multiagent Systems (AAMAS)},

address = {Auckland, New Zealand},

abstract = {This paper presents a computational model designed for planning both implicit and explicit communication of intentions, goals, and desires. Building upon previous research focused on implicit communication of intention via actions, our model seeks to strategically influence an observer’s belief using both the agent’s actions and explicit messages. We show that our proposed model can be considered to be a special case of general multi-agent problems with explicit communication under certain assumptions. Since the mental state of the observer depends on histories, computing a policy for the proposed model amounts to optimizing a non-Markovian objective, which we show to be intractable in the worst case. To mitigate this challenge, we propose a technique based on splitting domain and communication actions during planning. We conclude with experimental evaluations of the proposed approach that illustrate its effectiveness.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MVWZaamas24,

title = {Explaining the Behavior of POMDP-based Agents Through the Impact of Counterfactual Information},

author = {Saaduddin Mahmud and Marcell Vazquez-Chanlatte and Stefan Witwicki and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MVWZaamas24.pdf},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the The 23rd International Conference on Autonomous Agents and Multiagent Systems (AAMAS)},

address = {Auckland, New Zealand},

abstract = {In this work, we consider AI agents operating in Partially Observable Markov Decision Processes (POMDPs)–a widely-used framework for sequential decision making with incomplete state information. Agents operating with partial information take actions not only to advance their underlying goals but also to seek information and reduce uncertainty. Despite rapid progress in explainable AI, research on separating information-driven vs. goal-driven behaviors remains sparse. To address this gap, we introduce a novel explanation generation framework called Sequential Information Probing (SIP), to investigate the direct impact of state information, or its absence, on agent behavior. To quantify the impact we also propose two metrics under this SIP framework called Value of Information (VoI) and Influence of Information (IoI). We then theoretically derive several properties of these metrics. Finally, we present several experiments, including a case study on an autonomous vehicle, that illustrate the efficacy of our method.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:CSZZaamas24,

title = {Minimizing Negative Side Effects in Cooperative Multi-Agent Systems Using Distributed Coordination},

author = {Moumita Choudhury and Sandhya Saisubramanian and Hao Zhang and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/CSZZaamas24.pdf},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the The 23rd International Conference on Autonomous Agents and Multiagent Systems (AAMAS)},

address = {Auckland, New Zealand},

abstract = {Autonomous agents in real-world environments may encounter undesirable outcomes or negative side effects (NSEs) when working collaboratively alongside other agents. We frame the challenge of minimizing NSEs in a multi-agent setting as a lexicographic decentralized Markov decision process in which we assume independence of rewards and transitions with respect to the primary assigned tasks, but allowing negative side effects to create a form of dependence among the agents. We present a lexicographic Q-learning approach to mitigate the NSEs using human feedback models while maintaining near-optimality with respect to the assigned tasks–up to some given slack. Our empirical evaluation across two domains demonstrates that our collaborative approach effectively mitigates NSEs, outperforming non-collaborative methods.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:CSZZflairs24,

title = {Minimizing Negative Side Effects in Cooperative Multi-Agent Systems Using Distributed Coordination},

author = {Moumita Choudhury and Sandhya Saisubramanian and Hao Zhang and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/CSZZflairs24.pdf},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the The 37th International FLAIRS Conference},

address = {Miramar Beach, Florida},

abstract = {Autonomous agents operating in real-world environments frequently encounter undesirable outcomes or negative side effects (NSEs) when working collaboratively alongside other agents. Even when agents can execute their primary task optimally when operating in isolation, their training may not account for potential negative interactions that arise in the presence of other agents. We frame the challenge of minimizing NSEs as a lexicographic decentralized Markov decision process in which we assume independence of rewards and transitions with respect to the primary assigned tasks, but recognize that addressing negative side effects creates a form of dependence among the agents. We present a lexicographic Q-learning approach to mitigate the NSEs using human feedback models while maintaining near-optimality with respect to the assigned tasks–up to some given slack. Our empirical evaluation across two domains demonstrates that our collaborative approach effectively mitigates NSEs, outperforming non-collaborative methods.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:LSNZRicra24,

title = {Ethically Compliant Autonomous Systems under Partial Observability},

author = {Qingyuan Lu and Justin Svegliato and Samer B. Nashed and Shlomo Zilberstein and Stuart Russell},

url = {http://rbr.cs.umass.edu/shlomo/papers/LSNZRicra24.pdf},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},

address = {Yokohama, Japan},

abstract = {Ethically compliant autonomous systems (ECAS) are the prevailing approach to building robotic systems that perform sequential decision making subject to ethical theories in fully observable environments. However, in real-world robotics settings, these systems often operate under partial observability because of sensor limitations, environmental conditions, or limited inference due to bounded computational resources. Therefore, this paper proposes a partially observable ECAS (PO-ECAS), bringing this work one step closer to being a practical and useful tool for roboticists. First, we formally introduce the PO-ECAS framework and a MILP-based solution method for approximating an optimal ethically compliant policy. Next, we extend an existing ethical framework for prima facie duties to belief space and offer an ethical framework for virtue ethics inspired by Aristotle's Doctrine of the Mean. Finally, we demonstrate that our approach is effective in a simulated campus patrol robot domain.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NGZicra24,

title = {Choosing the Right Tool for the Job: Online Decision Making over SLAM Algorithms},

author = {Samer B. Nashed and Roderic A. Grupen and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/NGZicra24.pdf},

year  = {2024},

date = {2024-01-01},

booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},

address = {Yokohama, Japan},

abstract = {Nearly all state-of-the-art SLAM algorithms are designed to exploit patterns in data from specific sensing modalities, such as time-of-flight and structured light depth sensors, or RGB cameras. This specialization increases localization accuracy in domains where the given modality detects many high-quality features, but comes at the cost of decreasing performance in other, less favorable environments. For robotic systems that may experience a wide variety of sensing conditions, this difficulty in generalization presents a significant challenge. In this paper, we propose running several computationally cheap SLAM front ends in parallel and choosing the most promising feature set online. This problem is similar to the Algorithm Selection Problem (ASP), but has several complicating factors that preclude application of existing methods. We first provide an extension of the ASP formalism that captures the unique challenges in the SLAM setting, and then, based on this formalism, we propose modeling the SLAM ASP as a partially observable Markov decision process (POMDP). Our experiments show that dynamically selecting SLAM front ends, even myopically, improves localization robustness compared to selecting a static front end, and that using a POMDP policy provides even greater improvement.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SSPKZaaai23,

title = {Planning and Learning for Non-Markovian Negative Side Effects Using Finite State Controllers},

author = {Aishwarya Kamath and Sandhya Saisubramanian and Praveen Paruchuri and Akshat Kumar and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SSPKZaaai23.pdf},

year  = {2023},

date = {2023-01-01},

urldate = {2023-01-01},

booktitle = {Proceedings of the 37th Conference on Artificial Intelligence (AAAI)},

abstract = {Autonomous systems are often deployed in the open world where it is hard to obtain complete specifications of objectives and constraints. Operating based on an incomplete model can produce negative side effects (NSEs), which affect the safety and reliability of the system. We focus on mitigating NSEs in environments modeled as Markov decision processes (MDPs). First, we learn a model of NSEs using observed data that contains state-action trajectories and the severity of the associated NSEs. Unlike previous works that associate NSEs with state-action pairs, our framework associates NSEs with entire trajectories, which is more general and captures non-Markovian dependence on states and actions. Second, we learn finite state controllers (FSCs) that predict the NSE severity for a given trajectory and generalize well to unseen data. Finally, we develop a constrained MDP model that uses information from both the underlying MDP and the learned FSC for planning while avoiding NSEs. Our empirical evaluation demonstrates the effective- ness of our approach in learning and mitigating Markovian and non-Markovian NSEs.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BBZaaai23bridge,

title = {Competence-Aware Autonomy: An Essential Skill for Robots in the Real World},

author = {Connor Basich and Shlomo Zilberstein and Joydeep Biswas},

url = {http://rbr.cs.umass.edu/shlomo/papers/BBZaaai23bridge.pdf},

year  = {2023},

date = {2023-01-01},

booktitle = {Proceedings of the 37th Conference on Artificial Intelligence (AAAI) Bridge Program},

abstract = {Recent efforts in AI and robotics towards deploying intelligent robotic systems in the real world offer the possibility of transformational impacts on society. For such systems to be successful while reliably maintaining safe operation, they must be cognizant of their limitations, and when uncertain about their autonomous capabilities, solicit human assistance. However, system designers cannot fully enumerate the space of all situations that a robot deployed in the real world might face, prompting the challenge of endowing robots with actionable awareness of their capabilities and limitations in unseen settings. We propose competence-aware autonomy as a means of addressing this challenge in a well-defined manner motivated by real world examples. We discuss recent prior work in this area and suggest several research challenges and opportunities for future work.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MBZaamas23,

title = {Semi-Autonomous Systems with Contextual Competence Awareness},

author = {Saaduddin Mahmud and Connor Basich and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MBZaamas23.pdf},

year  = {2023},

date = {2023-01-01},

booktitle = {Proceedings of the 22nd International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

pages = {689–697},

abstract = {Competence modeling is critical for the efficient and safe operation of semi-autonomous systems (SAS) with varying levels of autonomy. In this paper, we extend the notion of competence modeling by introducing a contextual competence model. While previous work on competence-aware systems (CAS) defined the competence of a SAS relative to a single static operator, we present an augmented operator model that is contextualized by Markovian state information capable of capturing multiple operators. Access to such information allows the SAS to account for the stochastic shifts that may occur in the behavior of the operator(s) during deployment and optimize its autonomy accordingly. We show that the extended model called Contextual Competence Aware System (CoCAS) has the same convergence guarantees as CAS, and empirically illustrate the benefit of our approach over both the original CAS model as well as other relevant work in shared autonomy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NMGZaamas23,

title = {Causal Explanations for Sequential Decision Making Under Uncertainty (Extended Abstract)},

author = {Samer B. Nashed and Saaduddin Mahmud and Claudia V. Goldman and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/NMGZaamas23.pdf},

year  = {2023},

date = {2023-01-01},

booktitle = {Proceedings of the 22nd International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

pages = {2307–2309},

abstract = {Competence modeling is critical for the efficient and safe operation of semi-autonomous systems (SAS) with varying levels of autonomy. In this paper, we extend the notion of competence modeling by introducing a contextual competence model. While previous work on competence-aware systems (CAS) defined the competence of a SAS relative to a single static operator, we present an augmented operator model that is contextualized by Markovian state information capable of capturing multiple operators. Access to such information allows the SAS to account for the stochastic shifts that may occur in the behavior of the operator(s) during deployment and optimize its autonomy accordingly. We show that the extended model called Contextual Competence Aware System (CoCAS) has the same convergence guarantees as CAS, and empirically illustrate the benefit of our approach over both the original CAS model as well as other relevant work in shared autonomy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MSZijcai23,

title = {Explanation-Guided Reward Alignment},

author = {Saaduddin Mahmud and Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MSZijcai23.pdf},

year  = {2023},

date = {2023-01-01},

urldate = {2020-01-01},

booktitle = {Proceedings of the 32nd International Joint Conference on Artificial Intelligence (IJCAI)},

abstract = {Agents often need to infer a reward function from observations in order to learn desired behaviors. However, agents may infer a reward function that does not align with the original intent, as there can be multiple reward functions consistent with their observations. Operating based on such misaligned rewards can be risky. Furthermore, black-box representations make it difficult to verify the learned reward functions and prevent harmful behavior. We present a framework for verifying and improving reward alignment using explanations, and we show how explanations can help detect misalignment and reveal failure cases in novel scenarios. The problem is formulated as inverse reinforcement learning from ranked trajectories. Verification tests created from the trajectory dataset are used to iteratively verify and improve reward alignment. The agent explains its learned reward, and a tester signals whether the explanation passes the test. In cases where the explanation fails, the agent offers alternative explanations to gather feedback, which is then used to improve the learned reward. We analyze the efficiency of our approach in improving reward alignment using different types of explanations and demonstrate its effectiveness in five domains.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BNZgenplan23,

title = {RL3: Boosting Meta Reinforcement Learning via RL inside RL2},

author = {Abhinav Bhatia and Samer B. Nashed and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BNZgenplan23.pdf},

year  = {2023},

date = {2023-01-01},

urldate = {2023-01-01},

booktitle = {NeurIPS Workshop on Generalized Planning (GenPlan)},

address = {New Orleans, Louisiana},

abstract = {Meta reinforcement learning (meta-RL) methods such as RL2 have emerged as promising approaches for learning data-efficient RL algorithms tailored to a given task distribution. However, these RL algorithms struggle with long-horizon tasks and out-of-distribution tasks since they rely on recurrent neural networks to pro- cess the sequence of experiences instead of summarizing them into general RL components such as value functions. Moreover, even transformers have a practical limit to the length of histories they can efficiently reason about before training and inference costs become prohibitive. In contrast, traditional RL algorithms are data-inefficient since they do not leverage domain knowledge, but they do converge to an optimal policy as more data becomes available. In this paper, we propose RL3, a principled hybrid approach that combines traditional RL and meta-RL by incorporating task-specific action-values learned through traditional RL as an input to the meta-RL neural network. We show that RL3 earns greater cumulative reward on long-horizon and out-of-distribution tasks compared to RL2, while maintaining the efficiency of the latter in the short term. Experiments are conducted on both custom and benchmark discrete domains from the meta-RL literature that exhibit a range of short-term, long-term, and complex dependencies.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BMZiros23,

title = {Learning Constraints on Autonomous Behavior from Proactive Feedback},

author = {Connor Basich and Sadduddin Mahmud and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BMZiros23.pdf},

year  = {2023},

date = {2023-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {3680–3687},

address = {Detroit, Michigan},

abstract = {Learning from feedback is a common paradigm to acquire information that is hard to specify a priori. In this work, we consider a planning agent with a known nominal reward model that captures their high-level task objective, but is subject to constraints that are unknown a priori and must be inferred from human interventions. Unlike existing methods, our approach does not rely on full or partial demonstration trajectories or assume a fully reactive human. Instead, we assume access only to sparse interventions, which may in fact be generated proactively by the human, and make only minimal assumptions about the human. We provide both theoretical bounds on performance, and empirical validations of our method. We show that our method enables an agent to learn a constraint set with high accuracy that generalizes well to new environments within a domain, whereas methods that only consider reactive feedback learn an incorrect constraint set that does not generalize well, making constraint violations more likely in new environments.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NSNZRiros23,

title = {Formal Composition of Robotic Systems as Contract Programs},

author = {Mason Nakamura and Justin Svegliato and Samer B. Nashed and Shlomo Zilberstein and Stuart Russell},

url = {http://rbr.cs.umass.edu/shlomo/papers/NSNZRiros23.pdf},

year  = {2023},

date = {2023-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {6727–6732},

address = {Detroit, Michigan},

abstract = {Robotic systems are often composed of modular algorithms that each perform a specific function within a larger architecture, ranging from state estimation and task planning to trajectory optimization and object recognition. Existing work for specifying these systems as a formal composition of contract algorithms has limited expressiveness compared to the variety of sophisticated architectures that are commonly used in practice. Therefore, in this paper, we (1) propose a novel metareasoning framework for formally composing robotic systems as a contract program with programming constructs for functional, conditional, and looping semantics and (2) introduce a recursive hill climbing algorithm that finds a locally optimal time allocation of a contract program. In our experiments, we demonstrate that our approach outperforms baseline techniques in a simulated pick-and-place robot domain.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SBSZicra22,

title = {Metareasoning for Safe Decision Making in Autonomous Systems},

author = {Justin Svegliato and Connor Basich and Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SBSZicra22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},

address = {Philadelphia, Pennsylvania},

abstract = {Although experts carefully specify the high-level decision-making models in autonomous systems, it is infeasible to guarantee safety across every scenario during operation. We therefore propose a safety metareasoning system that optimizes the severity of the system's safety concerns and the interference to the system's task: the system executes in parallel a task process that completes a specified task and safety processes that each address a specified safety concern with a conflict resolver for arbitration. This paper offers a formal definition of a safety metareasoning system, a recommendation algorithm for a safety process, an arbitration algorithm for a conflict resolver, an application of our approach to planetary rover exploration, and a demonstration that our approach is effective in simulation.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BSNZicaps22,

title = {Tuning the Hyperparameters of Anytime Planning: A Metareasoning Approach with Deep Reinforcement Learning},

author = {Abhinav Bhatia and Justin Svegliato and Samer B. Nashed and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BSNZicaps22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the 32nd International Conference on Automated Planning and Scheduling (ICAPS)},

address = {Virtual Conference},

abstract = {Anytime planning algorithms often have hyperparameters that can be tuned at runtime to optimize their performance. While work on metareasoning has focused on when to interrupt an anytime planner and act on the current plan, the scope of metareasoning can be expanded to tuning the hyperparameters of the anytime planner at runtime. This paper introduces a general, decision-theoretic metareasoning approach that optimizes both the stopping point and hyperparameters of any- time planning. We begin by proposing a generalization of the standard meta-level control problem for anytime algorithms. We then offer a meta-level control technique that monitors and controls an anytime algorithm using deep reinforcement learning. Finally, we show that our approach boosts performance on a common benchmark domain that uses anytime weighted A* to solve a range of heuristic search problems and a mobile robot application that uses RRT* to solve motion planning problems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MWZsocs22,

title = {Heuristic Search for SSPs with Lexicographic Preferences over Multiple Costs},

author = {Shuwa Miura and Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MWZsocs22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the 15th Annual Symposium on Combinatorial Search (SOCS)},

address = {Vienna, Austria},

abstract = {Real-world decision problems often involve multiple competing objectives. The Stochastic Shortest Path (SSP) with lexicographic preferences over multiple costs offers an expressive formulation for many practical problems. However, the existing solution methods either lack optimality guarantees or require costly computations over the entire state space. We propose the first heuristic search algorithm for this problem, based on the heuristic algorithm for Constrained SSPs. Our experiments show that our heuristic search algorithm can compute optimal policies while avoiding a large portion of the state space. We also analyze the theoretical properties of the problem, establishing the conditions under which SSPs with lexicographic preferences have a proper optimal policy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PKKKZsocs22,

title = {Trajectory Constraint Heuristics for Optimal Probabilistic Planning},

author = {John R. Peterson and Anagha Kulkarni and Emil Keyder and Joseph Kim and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PKKKZsocs22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the 15th Annual Symposium on Combinatorial Search (SOCS)},

address = {Vienna, Austria},

abstract = {Search algorithms such as LAO* and LRTDP coupled with admissible heuristics are widely used methods for optimal probabilistic planning. Their effectiveness depends on the degree to which heuristics are able to approximate the optimal cost of a state. Many common domain-independent heuristics, however, rely on determinization, and ignore the probabilities associated with different effects of actions. Here, we present a method for decomposing a probabilistic planning problem into subproblems by constraining possible action outcomes. Admissible heuristics evaluated for each subproblem can then be combined via a weighted sum to obtain an admissible heuristic for the original problem that takes into account a limited amount of probabilistic information. We use this approach to derive new admissible heuristics for probabilistic planning, and show that for some problems they are significantly more informative than existing heuristics, giving up to an order of magnitude speedup in the time to converge to an optimal policy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BRCZiros22,

title = {A Sampling Based Approach to Robust Planning for a Planetary Lander},

author = {Connor Basich and Joseph A. Russino and Steve Chien and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BRCZiros22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {4106--4111},

address = {Kyoto, Japan},

abstract = {Planning for autonomous operation in unknown environments poses a number of technical challenges. The agent must ensure robustness to unknown phenomena, un- predictable variation in execution, and uncertain resources, all while maximizing its objective. These challenges are ex- acerbated in the context of space missions where uncertainty is often higher, long communication delays necessitate robust autonomous execution, and severely constrained computational resources limit the scope of planning techniques that can be used. We examine this problem in the context of a Europa Lander concept mission where an autonomous lander must collect valuable data and communicate that data back to Earth. We model the problem as a hierarchical task network, framing it as a utility maximization problem constrained by a strictly monotonically decreasing energy resource. We propose a novel deterministic planning framework that uses periodic replanning and sampling-based optimization to better handle model uncertainty and execution variation, while remaining computationally tractable. We demonstrate the efficacy of our framework through simulations of a Europa Lander concept mission in which our approach outperforms several baselines in utility maximization and robustness.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BPZiros22,

title = {Planning with Intermittent State Observability: Knowing When to Act Blind},

author = {Connor Basich and John Peterson and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BPZiros22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {11657--11664},

address = {Kyoto, Japan},

abstract = {Contemporary planning models and methods often rely on constant availability of free state information at each step of execution. However, autonomous systems are increasingly deployed in the open world where state information may be costly or simply unavailable in certain situations. Failing to account for sensor limitations may lead to costly behavior or even catastrophic failure. While the partially observable Markov decision process (POMDP) can be used to model this problem, solving POMDPs is often intractable. We introduce a planning model called a semi-observable Markov decision process (SOMDP) specifically designed for MDPs where state observability may be intermittent. We propose an approach for solving SOMDPs that uses memory states to proactively plan for the potential loss of sensor information while exploiting the unique structure of SOMDPs. Our theoretical analysis and empirical evaluation demonstrate the advantages of SOMDPs relative to existing planning models.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NSBRZiros22,

title = {Selecting the Partial State Abstractions of MDPs: A Metareasoning Approach with Deep Reinforcement Learning},

author = {Samer B. Nashed and Justin Svegliato and Abhinav Bhatia and Stuart Russell and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/NSBRZiros22.pdf},

year  = {2022},

date = {2022-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {116665--11670},

address = {Kyoto, Japan},

abstract = {Markov decision processes (MDPs) are a common general-purpose model used in robotics for representing sequential decision-making problems. Given the complexity of robotics applications, a popular approach for approximately solving MDPs relies on state aggregation to reduce the size of the state space but at the expense of policy fidelity--offering a trade-off between policy quality and computation time. Naturally, this poses a challenging metareasoning problem: how can an autonomous system dynamically select different state abstractions that optimize this trade-off as it operates online? In this paper, we formalize this metareasoning problem with a notion of time-dependent utility and solve it using deep reinforcement learning. To do this, we develop several general, cheap heuristics that summarize the reward structure and transition topology of the MDP at hand to serve as effective features. Empirically, we demonstrate that our metareasoning approach outperforms several baseline approaches and a strong heuristic approach on a standard benchmark domain.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SNZaaai21,

title = {Ethically Compliant Sequential Decision Making},

author = {Justin Svegliato and Samer B Nashed and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SNZaaai21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the 35th Conference on Artificial Intelligence (AAAI)},

pages = {11657--11665},

abstract = {Enabling autonomous systems to comply with an ethical theory is critical given their accelerating deployment in domains that impact society. While many ethical theories have been studied extensively in moral philosophy, they are still challenging to implement by developers who build autonomous systems. This paper proposes a novel approach for building ethically compliant autonomous systems that optimize completing a task while following an ethical framework. First, we introduce a definition of an ethically compliant autonomous system and its properties. Next, we offer a range of ethical frameworks for divine command theory, prima facie duties, and virtue ethics. Finally, we demonstrate the accuracy and usability of our approach in a set of autonomous driving simulations and a user study of planning and robotics experts.},

note = {Distinguished Paper Award},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BSZsocs21,

title = {On the Benefits of Randomly Adjusting Anytime Weighted A*},

author = {Abhinav Bhatia and Justin Svegliato and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BSZsocs21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the 14th International Symposium on Combinatorial Search (SOCS)},

abstract = {Anytime Weighted A*--an anytime heuristic search algorithm that uses a weight to scale the heuristic value of each node in the open list--has proven to be an effective way to manage the trade-off between solution quality and computation time in heuristic search. Finding the best weight, however, is challenging because it depends on not only the characteristics of the domain and the details of the instance at hand, but also the available computation time. We propose a randomized version of this algorithm, called Randomized Weighted A*, that randomly adjusts its weight at runtime and show a counterintuitive phenomenon: RWA* generally per- forms as well or better than AWA* with the best static weight on a range of benchmark problems. The result is a simple algorithm that is easy to implement and performs consistently well without any offline experimentation or parameter tuning.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NSBBGZicra21,

title = {Solving Markov Decision Processes with Partial State Abstractions},

author = {Samer B Nashed and Justin Svegliato and Matteo Brucato and Connor Basich and Roderic A Grupen and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/NSBBGZicra21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},

abstract = {Autonomous systems often use approximate planners that exploit state abstractions to solve large MDPs in real-time decision-making problems. However, these planners can eliminate details needed to produce effective behavior in autonomous systems. We therefore propose a novel model, a partially abstract MDP, with a set of abstract states that each compress a set of ground states to condense irrelevant details and a set of ground states that expand from a set of grounded abstract states to retain relevant details. This paper offers (1) a definition of a partially abstract MDP that (2) generalizes its ground MDP and its abstract MDP and exhibits bounded optimality depending on its abstract MDP along with (3) a lazy algorithm for planning and execution in autonomous systems. The result is a scalable approach that computes near-optimal solutions to large problems in minutes rather than hours.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:GSZaies21,

title = {Learning to Generate Fair Clusters from Demonstrations},

author = {Sainyam Galhotra and Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/GSZaies21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society (AIES)},

abstract = {Fair clustering is the process of grouping similar entities together, while satisfying a mathematically well-defined fairness metric as a constraint. Due to the practical challenges in precise model specification, the prescribed fairness constraints are often incomplete and act as proxies to the intended fairness requirement. Clustering with proxies may lead to biased outcomes when the system is deployed. We examine how to identify the intended fairness constraint for a problem based on limited demonstrations from an expert. Each demonstration is a clustering over a subset of the data. We present an algorithm to identify the fairness metric from demonstrations and generate clusters using existing off-the-shelf clustering techniques, and analyze its theoretical properties. To extend our approach to novel fairness metrics for which clustering algorithms do not currently exist, we present a greedy method for clustering. Additionally, we investigate how to generate interpretable solutions using our approach. Empirical evaluation on three real-world datasets demonstrates the effectiveness of our approach in quickly identifying the underlying fairness and interpretability constraints, which are then used to generate fair and interpretable clusters.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NSZaies21,

title = {Ethically Compliant Planning within Moral Communities},

author = {Samer B Nashed and Justin Svegliato and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/NSZaies21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society (AIES)},

abstract = {Ethically compliant autonomous systems (ECAS) are the state-of- the-art for solving sequential decision-making problems under un- certainty while respecting constraints that encode ethical considerations. This paper defines a novel concept in the context of ECAS that is from moral philosophy, the moral community, which leads to a nuanced taxonomy of explicit ethical agents. We then propose new ethical frameworks that extend the applicability of ECAS to domains where a moral community is required. Next, we provide a formal analysis of the proposed ethical frameworks and conduct experiments that illustrate their differences. Finally, we discuss the implications of explicit moral communities that could shape research on standards and guidelines for ethical agents in order to better understand and predict common errors in their design and communicate their capabilities.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SRZchi21,

title = {Understanding User Attitudes Towards Negative Side Effects of AI Systems},

author = {Sandhya Saisubramanian and Shannon C Roberts and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SRZchi21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {CHI Conference on Human Factors in Computing Systems, Late-Breaking Work},

pages = {368:1--368:6},

abstract = {Artificial Intelligence (AI) systems deployed in the open world may produce negative side effects—which are unanticipated, undesirable outcomes that occur in addition to the intended outcomes of the system’s actions. These negative side effects affect users directly or indirectly, by violating their preferences or altering their environment in an undesirable, potentially harmful, manner. While the existing literature has started to explore techniques to overcome the impacts of negative side effects in deployed systems, there has been no prior efforts to determine how users perceive and respond to negative side effects. We surveyed 183 participants to develop an understanding of user attitudes towards side effects and how side effects impact user trust in the system. The surveys targeted two domains: an autonomous vacuum cleaner and an autonomous vehicle, each with 183 respondents. The results indicate that users are willing to tolerate side effects that are not safety-critical but prefer to minimize them as much as possible. Furthermore, users are willing to assist the system in mitigating negative side effects by providing feedback and reconfiguring the environment. Trust in the system diminishes if it fails to minimize the impacts of negative side effects over time. These results support key fundamental assumptions in existing techniques and facilitate the development of new methods to overcome negative side effects of AI systems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZaamas21,

title = {Mitigating Negative Side Effects via Environment Shaping (Extended Abstract)},

author = {Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZaamas21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the 20th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

abstract = {Agents operating in the open world often produce negative side effects (NSE), which are difficult to identify at design time. We examine how a human can assist an agent, beyond providing feedback, and exploit their broader scope of knowledge to mitigate the impacts of NSE. We formulate this problem as a human-agent team with decoupled objectives. The agent optimizes its assigned task, during which its actions may produce NSE. The human shapes the environment through minor reconfiguration actions so as to mitigate the impacts of agent's side effects, without significantly degrading agent performance. We present an algorithm to solve this problem. Empirical evaluation shows that the proposed framework can successfully mitigate NSE, without affecting the agent’s ability to complete its assigned task.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BSBWWZiros21,

title = {Improving Competence via Iterative State Space Refinement},

author = {Connor Basich and Justin Svegliato and Allyson Beach and Kyle Hollins Wray and Stefan J Witwicki and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BSBWWZiros21.pdf},

doi = {10.1109/IROS51168.2021.9636239},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {1865--1871},

address = {Prague, Czech Republic},

abstract = {Despite considerable efforts by human designers, accounting for every unique situation that an autonomous robotic system deployed in the real world could face is often an infeasible task. As a result, many such deployed systems still rely on human assistance in various capacities to complete certain tasks while staying safe. Competence-aware systems (CAS) is a recently proposed model for reducing such reliance on human assistance while in turn optimizing the system’s global autonomous operation by learning its own competence. However, such systems are limited by a fixed model of their environment and may perform poorly if their a priori planning model does not include certain features that emerge as important over the course of the system’s deployment. In this paper, we propose a method for improving the competence of a CAS over time by identifying important state features missing from the system’s model and incorporating them into its state representation, thereby refining its state space. Our approach exploits information that exists in the standard CAS model and adds no extra work to the human. The result is an agent that better predicts human involvement, improving its competence, reliability, and overall performance.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PKSZiros21,

title = {Agent-Aware State Estimation in Autonomous Vehicles},

author = {Shane Parr and Ishan Khatri and Justin Svegliato and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PKSZiros21.pdf},

doi = {10.1109/IROS51168.2021.9636210},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {6694--6699},

address = {Prague, Czech Republic},

abstract = {Autonomous systems often operate in environments where the behavior of multiple agents is coordinated by a shared global state. Reliable estimation of the global state is thus critical for successfully operating in a multi-agent setting. We introduce agent-aware state estimation--a framework for calculating indirect estimations of state given observations of the behavior of other agents in the environment. We also introduce transition-independent agent-aware state estimation--a tractable class of agent-aware state estimation--and show that it allows the speed of inference to scale linearly with the number of agents in the environment. As an example, we model traffic light classification in instances of complete loss of direct observation. By taking into account observations of vehicular behavior from multiple directions of traffic, our approach exhibits accuracy higher than that of existing traffic light-only HMM methods on a real-world autonomous vehicle data set under a variety of simulated occlusion scenarios.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BWRCZicaps21ws1,

title = {A Sampling-Based Optimization Approach to Handling Environmental Uncertainty for a Planetary Lander},

author = {Connor Basich and Daniel Wang and Joseph Russino and Steve Chien and Shlomo Zilberstein},

year  = {2021},

date = {2021-01-01},

booktitle = {ICAPS Workshop on Planning and Robotics (PlanRob)},

address = {Guangzhou, China},

abstract = {TBD.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MCZroman21,

title = {Maximizing Legibility in Stochastic Environments},

author = {Shuwa Miura and Andrew L Cohen and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MCZroman21.pdf},

doi = {10.1109/RO-MAN50785.2021.9515318},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the 30th IEEE International Conference on Robot & Human Interactive Communication, (RO-MAN)},

pages = {1053--1059},

address = {Vancouver, BC, Canada},

abstract = {Making an agent's intentions clear from its observed behavior is crucial for seamless human-agent interaction and for increased transparency and trust in AI systems. Existing methods that address this challenge and maximize legibility of behaviors are limited to deterministic domains. We develop a technique for maximizing legibility in stochastic environments and illustrate that using legibility as an objective improves interpretability of agent behavior in several scenarios. We provide initial empirical evidence that human subjects can better interpret legible behavior.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MZuai21,

title = {A Unifying Framework for Observer-Aware Planning and its Complexity},

author = {Shuwa Miura and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MZuai21.pdf},

year  = {2021},

date = {2021-01-01},

booktitle = {Proceedings of the 37th Conference on Uncertainty in Artificial Intelligence (UAI)},

pages = {610--620},

address = {Virtual Event},

abstract = {Being aware of observers and the inferences they make about an agent's behavior is crucial for successful multi-agent interaction. Existing works on observer-aware planning use different assumptions and techniques to produce observer-aware behaviors. We argue that observer-aware planning, in its most general form, can be modeled as an Interactive POMDP (I-POMDP), which requires complex modeling and is hard to solve. Hence, we introduce a less complex framework for producing observer-aware behaviors called Observer-Aware MDP (OAMDP) and analyze its relationship to I-POMDP. We establish the complexity of OAMDPs and show that they can improve interpretability of agent behaviors in several scenarios.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SKZijcai20,

title = {A Multi-Objective Approach to Mitigate Negative Side Effects},

author = {Sandhya Saisubramanian and Ece Kamar and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SKZijcai20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 29th International Joint Conference on Artificial Intelligence (IJCAI)},

abstract = {Agents operating in unstructured environments of- ten create negative side effects (NSE) that may not be easy to identify at design time. We examine how various forms of human feedback or autonomous exploration can be used to learn a penalty function associated with NSE during system deployment. We formulate the problem of mitigating the impact of NSE as a multi-objective Markov deci- sion process with lexicographic reward preferences and slack. The slack denotes the maximum deviation from an optimal policy with respect to the agent’s primary objective allowed in order to mitigate NSE as a secondary objective. Empirical evaluation of our approach shows that the proposed framework can successfully mitigate NSE and that different feedback mechanisms introduce different biases, which influence the identification of NSE.},

note = {Distinguished Paper Award},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SGZaies20,

title = {Balancing the Tradeoff Between Clustering Value and Interpretability},

author = {Sandhya Saisubramanian and Sainyam Galhotra and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SGZaies20.pdf},

doi = {10.1145/3375627.3375843},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society (AIES)},

pages = {351--357},

address = {New York, NY},

abstract = {Graph clustering groups entities -- the vertices of a graph -- based on their similarity, typically using a complex distance function over a large number of features. Successful integration of clustering approaches in automated decision-support systems hinges on the interpretability of the resulting clusters. This paper addresses the problem of generating interpretable clusters, given features of interest that signify interpretability to an end-user, by optimizing interpretability in addition to common clustering objectives. We propose a β-interpretable clustering algorithm that ensures that at least β fraction of nodes in each cluster share the same feature value. The tunable parameter β is user-specified. We also present a more efficient algorithm for scenarios with β = 1 and analyze the theoretical guarantees of the two algorithms. Finally, we empirically demonstrate the benefits of our approaches in generating interpretable clusters using four real-world datasets. The interpretability of the clusters is complemented by generating simple explanations denoting the feature values of the nodes in the clusters, using frequent pattern mining.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WGLZJCGspring20,

title = {AI-Enabled Training in Manufacturing Workforce Development},

author = {Beverly Woolf and Aritra Ghosh and Andrew Lan and Shlomo Zilberstein and Tom Juravich and Andrew Cohen and Olivia Geho},

year  = {2020},

date = {2020-01-01},

booktitle = {AAAI Spring Symposium on Artificial Intelligence in Manufacturing},

abstract = {A highly productive workforce can evolve with the integration of digital devices, such as computer interfaces to operating machines, interconnected smart devices, and robots, in the workplace. However, this potential cannot be realized with the current state-of-the-art systems used to train workers. This problem is acute in manufacturing, where huge skills gaps are evident; most workers lack the necessary skills to operate or collaborate with autonomous systems. We propose to address this problem by using intelligent tutoring systems and worker data analysis. The worker data includes: i) fine-grained on-job performance data, ii) career path data containing the entire career paths of workers, and iii) job posting data over a long period of time indicating the required skills for each job. We will collect and analyze worker data and use it to drive new methods for training and reskilling workers. We detail ideas and tools to be developed by research in intelligent tutoring systems, data science, manufacturing, sociology, labor analysis, education, psychology, and economics. We also describe a convergent approach to developing effective, fair, and scalable software solutions and dynamic intelligent training. address = Stanford, California},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BSWWBZaamas20,

title = {Learning to Optimize Autonomy in Competence-Aware Systems},

author = {Connor Basich and Justin Svegliato and Kyle Hollins Wray and Stefan J Witwicki and Joydeep Biswas and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BSWWBZaamas20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

address = {Auckland, New Zealand},

abstract = {Interest in semi-autonomous systems (SAS) is growing rapidly as a paradigm to deploy autonomous systems in domains that require occasional reliance on humans. This paradigm allows service robots or autonomous vehicles to operate at varying levels of autonomy and offer safety in situations that require human judgment. We propose an introspective model of autonomy that is learned and updated online through experience and dictates the extent to which the agent can act autonomously in any given situation. We define a competence-aware system (CAS) that explicitly models its own proficiency at different levels of autonomy and the available human feedback. A CAS learns to adjust its level of autonomy based on experience to maximize overall efficiency, factoring in the cost of human assistance. We analyze the convergence properties of CAS and provide experimental results for robot delivery and autonomous driving domains that demonstrate the benefits of the approach.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MZaamas20,

title = {Maximizing Plan Legibility in Stochastic Environments (Extended Abstract)},

author = {Shuwa Miura and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MZaamas20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

address = {Auckland, New Zealand},

abstract = {Legible behavior allows an observing agent to infer the intention of an observed agent. Producing legible behavior is crucial for successful multi-agent interaction in many domains. We introduce techniques for legible planning in stochastic environments. Maximizing legibility, however, presents a complex trade-off between maximizing the underlying rewards. Hence, we propose a method to balance the trade-off. In our experiments, we demonstrate that maximizing legibility results in unambiguous behaviors.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SKZaamas20,

title = {Mitigating the Negative Side Effects of Reasoning with Imperfect Models: A Multi-Objective Approach (Extended Abstract)},

author = {Sandhya Saisubramanian and Ece Kamar and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SKZaamas20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 19th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

address = {Auckland, New Zealand},

abstract = {Agents often operate using imperfect models of the environment that ignore certain aspects of the real world. Reasoning with such models may lead to negative side effects (NSE) when satisfying the primary objective of the available model, which are inherently difficult to identify at design time. We examine how various forms of feedback can be used to learn a penalty function associated with NSE during execution. We formulate the problem of mitigating the impact of NSE as a multi-objective Markov decision process with lexicographic reward preferences and slack. Empirical evaluation of our approach on three domains shows that the proposed framework can successfully mitigate NSE.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SSZicra20,

title = {A Model-Free Approach to Meta-Level Control of Anytime Algorithms},

author = {Justin Svegliato and Prakhar Sharma and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SSZicra20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},

address = {Paris, France},

abstract = {Anytime algorithms offer a trade-off between solution quality and computation time that has proven to be useful in autonomous systems for a wide range of real-time planning problems. In order to optimize this trade-off, an autonomous system has to solve a challenging meta-level control problem: it must decide when to interrupt the anytime algorithm and act on the current solution. Prevailing meta-level control techniques, however, make a number of unrealistic assumptions that reduce their effectiveness and usefulness in the real world. Eliminating these assumptions, we first introduce a model-free approach to meta-level control based on reinforcement learning and prove its optimality. We then offer a general meta-level control technique that can use different reinforcement learning methods. Finally, we show that our approach is effective across several common benchmark domains and a mobile robot domain.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WKGKYZecai20,

title = {Accounting for Observer's Partial Observability in Stochastic Goal Recognition Design},

author = {Christabel Wayllace and Sarah Keren and Avigdor Gal and Erez Karpas and William Yeoh and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WKGKYZecai20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 24th European Conference on Artificial Intelligence (ECAI)},

abstract = {Motivated by security applications, where agent intentions are unknown, actions may have stochastic outcomes, and an ob- server may have an obfuscated view due to low sensor resolution, we introduce partially-observable states and unobservable actions into a stochastic goal recognition design framework. The proposed model is accompanied by a method for calculating the expected maximal number of steps before the goal of an agent is revealed and a new sensor refinement modification that can be applied to enhance goal recognition. A preliminary empirical evaluation on a range of bench- mark applications shows the effectiveness of our approach.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PKSZicra20ws,

title = {Agent-Aware State Estimation: Effective Traffic Light Classification for Autonomous Vehicles},

author = {Shane Parr and Ishan Khatri and Justin Svegliato and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PKSZicra20ws.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {ICRA 2020 Workshop on Sensing, Estimating and Understanding the Dynamic World},

abstract = {Autonomous systems often operate in environments where the behavior of all agents is mostly governed by the perception of a specific feature of the environment. When an autonomous system cannot recover this feature, there can be disastrous consequences. We introduce a novel framework for agent-aware state estimation that exploits the dependency of all agents' behavior on a feature to better indirectly observe the feature. To allow for fast and accurate inference, we provide a mapping of our framework to a dynamic Bayesian network and show that speed of inference scales favorably with the number of agents in the environment. We then apply our approach to traffic light classification, focusing on instances where direct vision of the light may be obstructed by glare, heavy rain, vehicles, or other environmental factors. Finally, we show that agent-aware state estimation outperforms prevailing methods that only use direct image data of the traffic light on a real-world autonomous vehicle data set of challenging scenarios.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{FLWZfall20,

title = {Helpfulness as a Key Metric of Human-Robot Collaboration},

author = {Richard G Freedman and Steven J Levine and Brian C Williams and Shlomo Zilberstein},

url = {https://arxiv.org/pdf/2010.04914.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {AAAI Fall Symposium on Artificial Intelligence and Human-Robot Interaction (AI-HRI)},

abstract = {As robotic teammates become more common in society, people will assess the robots' roles in their interactions along many dimensions. One such dimension is effectiveness: people will ask whether their robotic partners are trustworthy and effective collaborators. This begs a crucial question: how can we quantitatively measure the helpfulness of a robotic partner for a given task at hand? This paper seeks to answer this question with regards to the interactive robot's decision making. We describe a clear, concise, and task-oriented metric applicable to many different planning and execution paradigms. The proposed helpfulness metric is fundamental to assessing the benefit that a partner has on a team for a given task. In this paper, we define helpfulness, illustrate it on concrete examples from a variety of domains, discuss its properties and ramifications for planning interactions with humans, and present preliminary results.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZJprima20,

title = {Multi-Agent Planning with High-Level Human Guidance},

author = {Feng Wu and Shlomo Zilberstein and Nicholas R Jennings},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZJprima20.pdf},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of Principles and Practice of Multi-Agent Systems (PRIMA)},

abstract = {Planning and coordination of multiple agents in the presence of uncertainty and noisy sensors is extremely hard. A human operator who observes a multi-agent team can provide valuable guidance to the team based on her superior ability to interpret observations and assess the overall situation. We propose an extension of decentralized POMDPs that allows such human guidance to be factored into the planning and execution processes. Human guidance in our framework consists of intuitive high-level commands that the agents must translate into a suitable joint plan that is sensitive to what they know from local observations. The result is a framework that allows multi-agent systems to benefit from the complex strategic thinking of a human supervising them. We evaluate this approach on several common benchmark problems and show that it can lead to dramatic improvement in performance.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZaaai19ws,

title = {Minimizing the Negative Side Effects of Planning with Reduced Models},

author = {Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZaaai19ws.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {AAAI Workshop on Artificial Intelligence Safety},

address = {Honolulu, Hawaii},

abstract = {Reduced models of large Markov decision processes accelerate planning by considering a subset of outcomes for each state-action pair. This reduction in reachable states leads to replanning when the agent encounters states without a pre-computed action during plan execution. However, not all states are suitable for replanning. In the worst case, the agent may not be able to reach the goal from the newly encountered state. Agents should be better prepared to handle such risky situations and avoid replanning in risky states. Hence, we consider replanning in states that are unsafe for deliberation as a negative side effect of planning with reduced models. While the negative side effects can be minimized by always using the full model, this defeats the purpose of using reduced models. The challenge is to plan with reduced models, but somehow account for the possibility of encountering risky situations. An agent should thus only replan in states that the user has approved as safe for replanning. To that end, we propose planning using a portfolio of reduced models, a planning paradigm that minimizes the negative side effects of planning using reduced models by alternating between different outcome selection approaches. We empirically demonstrate the effectiveness of our approach on three domains: an electric vehicle charging domain using real-world data from a university campus and two benchmark planning problems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:DFZWiccnc19,

title = {Using Natural Language Constructs and Concepts to Aid Network Management},

author = {Abhishek Dwaraki and Richard G Freedman and Shlomo Zilberstein and Tilman Wolf},

url = {https://doi.org/10.1109/ICCNC.2019.8685639},

doi = {10.1109/ICCNC.2019.8685639},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the International Conference on Computing, Networking and Communications},

pages = {802--808},

address = {Honolulu, Hawaii},

abstract = {The increasing complexity of networks together with technological trends that allow for fine-grained control and programmability have made network management a pressing challenge. In this work, we propose to harness the vast amounts of network management data that are available from different sources in an automated system that can infer context and semantics. We present an argument for a Network Processing Language that is based on the ideas of natural language processing. Our approach shows how concepts, such as collocations, can be applied to network management data. We demonstrate the effectiveness of our approach to detect route prefix and sub-prefix hijacks. This work presents one step toward effectively using automated tools for network management in complex, programmable networks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PZaamas19,

title = {Soft Labeling in Stochastic Shortest Path Problems},

author = {Luis Pineda and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PZaamas19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

pages = {467--475},

address = {Montreal, Quebec, CA},

abstract = {The Stochastic Shortest Path (SSP) is an established model for goal-directed probabilistic planning. Despite its broad applicability, wide adoption of the model has been impaired by its high computational complexity. Efforts to address this challenge have produced promising algorithms that leverage two popular mechanisms: labeling and short-sightedness. The resulting algorithms can generate near- optimal solutions much faster than optimal solvers, albeit at the cost of poor theoretical guarantees. In this work, we introduce a generalization of labeling, called soft labeling, which results in a framework that encompasses a wide spectrum of efficient labeling algorithms, and offers better theoretical guarantees than existing short-sighted labeling approaches. We also propose a novel instantiation of this framework, the SOFT-FLARES algorithm, which achieves state-of-the-art performance on a diverse set of benchmarks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZaamas19,

title = {Policy Networks: A Framework for Scalable Integration of Multiple Decision-Making Models (Extended Abstract)},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZaamas19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

pages = {2270--2272},

address = {Montreal, Quebec, CA},

abstract = {Policy networks are graphical models that integrate decision-making models. They allow for multiple Markov decision processes (MDPs) that describe distinct focused aspects of a domain to work in harmony to solve a large-scale problem. This paper defines policy networks and shows how they are able to naturally generalize many previous models, such as options and constrained MDPs.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZicra19,

title = {Generalized Controllers in POMDP Decision-Making},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZicra19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the IEEE International Conference on Robotics and Automation (ICRA)},

pages = {7166--7172},

address = {Montreal, Quebec, CA},

abstract = {We present a general policy formulation for partially observable Markov decision processes (POMDPs) called controller family policies that may be used as a framework to facilitate the design of new policy forms. We prove how modern approximate policy forms: point-based, finite state controller (FSC), and belief compression, are instances of this family of generalized controller policies. Our analysis provides a deeper understanding of the POMDP model and suggests novel ways to design POMDP solutions that can combine the benefits of different state-of-the-art methods. We illustrate this capability by creating a new customized POMDP policy form called the belief-integrated FSC (BI-FSC) tailored to overcome the shortcomings of a state-of-the-art algorithm that uses non-linear programming (NLP). Specifically, experiments show that for NLP the BI-FSC offers improved performance over a vanilla FSC-based policy form on benchmark domains. Furthermore, we demonstrate the BI-FSC's execution on a real robot navigating in a maze environment. Results confirm the value of using the controller family policy as a framework to design customized policies in POMDP robotic solutions.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KPGKZicaps19,

title = {Responsive Planning and Recognition for Closed-Loop Interaction},

author = {Sarah Keren and Luis Enrique Pineda and Avigdor Gal and Erez Karpas and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/KPGKZicaps19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 29th International Conference on Automated Planning and Scheduling (ICAPS)},

pages = {246--254},

address = {Berkeley, CA},

abstract = {Given an environment, the utility measure of the agents acting within it, a set of possible environment modifications, and a description of design constraints, the objective of equi-reward utility maximizing design (ER-UMD) is to find a valid sequence of modifications to apply to the environment in order to maximize agent utility. To efficiently traverse the typically large space of possible design options, we use heuristic search and propose new heuristics, which relax the design process; instead of computing the value achieved by a single modification, we use a dominating modification guaranteed to be at least as beneficial. The proposed technique enables heuristic caching for similar nodes thereby saving computational overhead. We specify sufficient conditions under which our approach is guaranteed to produce admissible estimates, and describe a range of models that comply with these requirements. Also, for models with lifted representations of environment modifications, we provide simple methods to automatically generate dominating modifications. We evaluate our approach on a range of stochastic settings for which our heuristic is admissible. We demonstrate its efficiency by comparing it to a previously suggested heuristic that employs a relaxation of the environment, and to a compilation from ER-UMD to planning.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SWPZicaps19ws1,

title = {Planning in Stochastic Environments with Goal Uncertainty},

author = {Sandhya Saisubramanian and Kyle Hollins Wray and Luis Enrique Pineda and Shlomo Zilberstein},

year  = {2019},

date = {2019-01-01},

booktitle = {ICAPS Workshop on Planning and Robotics (PlanRob)},

address = {Berkeley, CA},

abstract = {TBD.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SBZGicaps19ws2,

title = {The Value of Incorporating Social Preferences in Dynamic Ridesharing},

author = {Sandhya Saisubramanian and Connor Basich and Shlomo Zilberstein and Claudia V Goldman},

year  = {2019},

date = {2019-01-01},

booktitle = {ICAPS Workshop on Scheduling and Planning Applications (SPARK)},

address = {Berkeley, CA},

abstract = {TBD.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SWPZiros19,

title = {Planning in Stochastic Environments with Goal Uncertainty},

author = {Sandhya Saisubramanian and Kyle Hollins Wray and Luis Enrique Pineda and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SWPZiros19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

address = {Macau, China},

abstract = {We present the Goal Uncertain Stochastic Shortest Path (GUSSP) problem -- a general framework to model path planning and decision making in stochastic environments with goal uncertainty. The framework extends the stochastic shortest path (SSP) model to dynamic environments in which it is impossible to determine the exact goal states ahead of plan execution. GUSSPs introduce flexibility in goal specification by allowing a belief over possible goal configurations. The unique observations at potential goals helps the agent identify the true goal during plan execution. The partial observability is restricted to goals, facilitating the reduction to an SSP with a modified state space. We formally define a GUSSP and discuss its theoretical properties. We then propose an admissible heuristic that reduces the planning time using FLARES -- a start-of-the-art probabilistic planner. We also propose a determinization approach for solving this class of problems. Finally, we present empirical results on a search and rescue mobile robot and three other problem domains in simulation.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZiros19,

title = {Adaptive Outcome Selection for Planning With Reduced Models},

author = {Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZiros19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

address = {Macau, China},

abstract = {Reduced models allow autonomous robots to cope with the complexity of planning in stochastic environments by simplifying the model and reducing its accuracy. The solution quality of a reduced model depends on its fidelity. We present 0/1 reduced model that selectively improves model fidelity in certain states by switching between using a simplified deterministic model and the full model, without significantly compromising the run time gains. We measure the reduction impact for a reduced model based on the values of the ignored outcomes and use this as a heuristic for outcome selection. Finally, we present empirical results of our approach on three different domains, including an electric vehicle charging problem using real-world data from a university campus.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SWWBZiros19,

title = {Belief Space Metareasoning for Exception Recovery},

author = {Justin Svegliato and Kyle Hollins Wray and Stefan J Witwicki and Joydeep Biswas and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SWWBZiros19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

address = {Macau, China},

abstract = {Due to the complexity of the real world, autonomous systems use decision-making models that rely on simplifying assumptions to make them computationally tractable and feasible to design. However, since these limited representations cannot fully capture the domain of operation, an autonomous system may encounter unanticipated scenarios that cannot be resolved effectively. We first formally introduce an introspective autonomous system that uses belief space metareasoning to recover from exceptions by interleaving a main decision process with a set of exception handlers. We then apply introspective autonomy to autonomous driving. Finally, we demonstrate that an introspective autonomous vehicle is effective in simulation and on a fully operational prototype.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SBZGitsc19,

title = {Satisfying Social Preferences in Ridesharing Services},

author = {Sandhya Saisubramanian and Connor Basich and Shlomo Zilberstein and Claudia V Goldman},

url = {http://rbr.cs.umass.edu/shlomo/papers/SBZGitsc19.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the IEEE Intelligent Transportation Systems Conference (ITSC)},

pages = {3720--3725},

address = {Auckland, New Zealand},

abstract = {Dynamic ridesharing services (DRS) play a major role in improving the efficiency of urban transportation. User satisfaction in dynamic ridesharing is determined by multiple factors such as travel time, cost, and social compatibility with co-passengers. Existing DRS optimize profit by maximizing the operational value for service providers or minimizing the travel time for users but they neglect the social experience of riders, which significantly influences the total value of the service to users. We propose DROPS, a dynamic ridesharing framework that factors the riders’ social preferences in the matching process so as to improve the quality of the trips formed. The trip formation is a multi-objective optimization that aims to maximize the operational value for the service provider, while simultaneously maximizing the value of the trip for the users. The user value is estimated based on compatibility between co-passengers and the ride time. We also present a real-time matching algorithm for trip formation. Finally, we evaluate our approach empirically using real-world taxi trips data, and a population model including social preferences based on user surveys. Our approach improves the user value and users’ social compatibility, without significantly affecting the vehicle miles for the service provider and travel time for users.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZJdai19,

title = {Stochastic Multi-agent Planning with Partial State Models},

author = {Feng Wu and Shlomo Zilberstein and Nicholas R Jennings},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZJdai19.pdf},

doi = {10.1145/3356464.3357699},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the First International Conference on Distributed Artificial Intelligence (DAI)},

pages = {1-8},

address = {Beijing, China},

abstract = {People who observe a multi-agent team can often provide valuable information to the agents based on their superior cognitive abilities to interpret sequences of observations and assess the overall situation. The knowledge they possess is often difficult to be fully represented using a formal model such as DEC-POMDP. To deal with this, we propose an extension of the DEC-POMDP that allows states to be partially specified and benefit from expert knowledge, while preserving the partial observability and decentralized operation of the agents. In particular, we present an algorithm for computing policies based on history samples that include human labeled data in the form of reward reshaping. We also consider ways to minimize the burden on human experts during the labeling phase. The results offer the first approach to incorporating human knowledge in such complex multi-agent settings. We demonstrate the benefits of our approach using a disaster recovery scenario, comparing it to several baseline approaches.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZCaaai18,

title = {Privacy-Preserving Policy Iteration for Decentralized POMDPs},

author = {Feng Wu and Shlomo Zilberstein and Xiaoping Chen},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZCaaai18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 32nd Conference on Artificial Intelligence (AAAI)},

pages = {4759--4766},

address = {New Orleans, Louisiana},

abstract = {We propose the first privacy-preserving approach to address the privacy issues that arise in multi-agent planning problems modeled as a Dec-POMDP. Our solution is a distributed message-passing algorithm based on trials, where the agents' policies are optimized using the cross-entropy method. In our algorithm, the agents' private information is protected using a public-key homomorphic cryptosystem. We prove the correctness of our algorithm and analyze its complexity in terms of message passing and encryption/decryption operations. Furthermore, we analyze several privacy aspects of our algorithm and show that it can preserve the agent privacy of non-neighbors, model privacy, and decision privacy. Our experimental results on several common Dec-POMDP bench- mark problems confirm the effectiveness of our approach.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WKZaaai18,

title = {Integrated Cooperation and Competition in Multi-Agent Decision-Making},

author = {Kyle Hollins Wray and Akshat Kumar and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WKZaaai18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 32nd Conference on Artificial Intelligence (AAAI)},

pages = {4751--4758},

address = {New Orleans, Louisiana},

abstract = {Observing that many real-world sequential decision problems are not purely cooperative or purely competitive, we propose a new model--cooperative-competitive process (CCP)--that can simultaneously encapsulate both cooperation and competition. First, we discuss how the CCP model bridges the gap between cooperative and competitive models. Next, we investigate a specific class of group-dominant CCPs, in which agents cooperate to achieve a common goal as their primary objective, while also pursuing individual goals as a secondary objective. We provide an approximate solution for this class of problems that leverages stochastic finite-state controllers. The model is grounded in two multi-robot meeting and box-pushing domains that are implemented in simulation and demonstrated on two real robots.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FFGZpair18,

title = {Towards Quicker Probabilistic Recognition with Multiple Goal Heuristic Search},

author = {Richard G Freedman and Yi Ren Fung and Roman Ganchin and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FFGZpair18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {AAAI Workshop on Plan, Activity, and Intent Recognition (PAIR)},

pages = {601--606},

address = {New Orleans, Louisiana},

abstract = {Referred to as an approach for either plan or goal recognition, the original method proposed by Ramirez and Geffner introduced a domain-based approach that did not need a library containing specific plan instances. This introduced a more generalizable means of representing tasks to be recognized, but was also very slow due to its need to run simulations via multiple executions of an off-the-shelf classical planner. Several variations have since been proposed for quicker recognition, but each one uses a drastically different approach that must sacrifice other qualities useful for processing the recognition results in more complex systems. We present work in progress that takes advantage of the shared state space between planner executions to perform multiple goal heuristic search. This single execution of a planner will potentially speed up the recognition process using the original method, which also maintains the sacrificed properties and improves some of the assumptions made by Ramirez and Geffner.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FZkeg18,

title = {Roles that Plan, Activity, and Intent Recognition with Planning Can Play in Games},

author = {Richard G Freedman and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FZkeg18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {AAAI Workshop on Knowledge Extraction from Games (KEG)},

address = {New Orleans, Louisiana},

abstract = {Planning is one of the oldest areas of research within artificial intelligence, studying the selection of actions for accomplishing goals. The more recently established areas of plan, activity, and intent recognition instead study an agent's behavior and task(s) given observations of its chosen actions. While these areas have been independently studied and applied to games in the past for both understanding player behavior and developing game characters, the potential for their integration presents even more opportunities via adaptive interaction with the player. In this manuscript, we discuss recent research on the integration of these areas and investigate potential uses for such integrated systems in games.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SDFZicaps18ws1,

title = {An Anytime Algorithm for Task and Motion MDPs},

author = {Siddharth Srivastava and Nishant Desai and Richard G Freedman and Shlomo Zilberstein},

url = {http://arxiv.org/abs/1802.05835},

year  = {2018},

date = {2018-01-01},

booktitle = {ICAPS Workshop on Planning and Robotics (PlanRob)},

address = {Delft, The Netherlands},

abstract = {Integrated task and motion planning has emerged as a challenging problem in sequential decision making, where a robot needs to compute high-level strategy and low-level motion plans for solving complex tasks. While high-level strategies require decision making over longer time-horizons and scales, their feasibility depends on low-level constraints based upon the geometries and continuous dynamics of the environment. The hybrid nature of this problem makes it difficult to scale; most existing approaches focus on deterministic, fully observable scenarios. We present a new approach where the high-level decision problem occurs in a stochastic setting and can be modeled as a Markov decision process. In contrast to prior efforts, we show that complete MDP policies, or contingent behaviors, can be computed effectively in an anytime fashion. Our algorithm continuously improves the quality of the solution and is guaranteed to be probabilistically complete. We evaluate the performance of our approach on a challenging, realistic test problem: autonomous aircraft inspection. Our results show that we can effectively compute consistent task and motion policies for the most likely execution-time outcomes using only a fraction of the computation required to develop the complete task and motion policy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SDFZicaps18ws2,

title = {Relaxed Modification Heuristics for Equi-Reward Utility Maximizing Design},

author = {Sarah Keren and Luis Enrique Pineda and Avigdor Gal and Erez Karpas and Shlomo Zilberstein},

year  = {2018},

date = {2018-01-01},

booktitle = {ICAPS Workshop on Heuristics and Search for Domain-Independent Planning (HSDIP)},

address = {Delft, The Netherlands},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZSaamas18,

title = {Planning Using a Portfolio of Reduced Models},

author = {Sandhya Saisubramanian and Shlomo Zilberstein and Prashant J Shenoy},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZSaamas18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 17th International Conference on Autonomous Agents and MultiAgent Systems (AAMAS)},

pages = {2057--2059},

address = {Stockholm, Sweden},

abstract = {Existing reduced model techniques simplify a problem by applying a uniform principle to reduce the number of considered outcomes for all state-action pairs. It is non-trivial to identify which outcome selection principle will work well across all problem instances in a domain. We aim to create reduced models that yield near-optimal solutions, without compromising the run time gains of using a reduced model. First, we introduce planning using a portfolio of reduced models, a framework that provides flexibility in the reduced model formulation by using a portfolio of outcome selection principles. Second, we propose planning using cost adjustment, a technique that improves the solution quality by accounting for the outcomes ignored in the reduced model. Empirical evaluation of these techniques confirm their effectiveness in several domains.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZijcaiPLW18,

title = {Safe Reduced Models for Probabilistic Planning},

author = {Sandhya Saisubramanian and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZijcaiPLW18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {ICML/IJCAI/AAMAS Workshop on Planning and Learning (PAL)},

address = {Stockholm, Sweden},

abstract = {Reduced models allow autonomous agents to cope with the complexity of planning under uncertainty by reducing the accuracy of the model. However, the solution quality of a reduced model varies as the model fidelity changes. We present planning using a portfolio of reduced models with cost adjustments, a framework to increase the safety of a reduced model by selectively improving its fidelity in certain states, without significantly compromising runtime. Our framework provides the flexibility to create reduced models with different levels of de- tail using a portfolio, and a means to account for the ignored details by adjusting the actions costs in the reduced model. We show the conditions under which cost adjustments achieve optimal action selection and describe how to use cost adjustments as a heuristic for choosing outcome selection principles in a portfolio. Finally, we present empirical results of our approach on three domains that includes an electric vehicle charging problem using real-world data from a university campus.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SWZijcai18,

title = {Meta-Level Control of Anytime Algorithms with Online Performance Prediction},

author = {Justin Svegliato and Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SWZijcai18.pdf},

doi = {10.24963/ijcai.2018/208},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 27th International Joint Conference on Artificial Intelligence},

pages = {1499--1505},

address = {Stockholm, Sweden},

abstract = {Anytime algorithms enable intelligent systems to trade computation time with solution quality. To exploit this crucial ability in real-time decision-making, the system must decide when to interrupt the anytime algorithm and act on the current solution. Existing meta-level control techniques, how- ever, address this problem by relying on significant offline work that diminishes their practical utility and accuracy. We formally introduce an online performance prediction framework that enables meta- level control to adapt to each instance of a problem without any preprocessing. Using this framework, we then present a meta-level control technique and two stopping conditions. Finally, we show that our approach outperforms existing techniques that re- quire substantial offline work. The result is efficient nonmyopic meta-level control that reduces the overhead and increases the benefits of using any- time algorithms in intelligent systems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SWHZijcaiAI4IoT18,

title = {Belief-Space Planning for Automated Malware Defense},

author = {Justin Svegliato and Sam Witty and Amir Houmansadr and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SWHZijcaiAI4IoT18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {IJCAI/ECAI Workshop on AI for Internet of Things (AI4IoT)},

address = {Stockholm, Sweden},

abstract = {Malware detection and response is critical to ensuring information security across a wide range of devices. There have been few attempts, however, to develop security systems that exploit the benefits of different malware detection techniques. We formally introduce an automated malware defense framework and represent it as a belief-space planning problem that optimally reduces the impact on the performance of a system. Using the framework, we then provide an example automated malware defense system for email worm detection and response. Finally, we show in simulation that the system outperforms standard security techniques that have been used in practice. The result is a novel belief-space planning approach to auto- mated malware defense designed for robust, accurate, and efficient use in large networks of resource-constrained devices.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZijcaiAEGAP18,

title = {Adaptive Metareasoning for Bounded Rational Agents},

author = {Justin Svegliato and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZijcaiAEGAP18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {IJCAI/ECAI Workshop on Architectures and Evaluation for Generality, Autonomy and Progress in AI (AEGAP)},

address = {Stockholm, Sweden},

abstract = {In computational approaches to bounded rationality, metareasoning enables intelligent agents to optimize their own decision-making process in order to produce effective action in a timely manner. While there have been substantial efforts to develop effective meta-level control for anytime algorithms, existing techniques rely on extensive offline work, imposing several critical assumptions that diminish their effectiveness and limit their practical utility in the real world. In order to eliminate these assumptions, adaptive metareasoning enables intelligent agents to adapt to each individual instance of the problem at hand without the need for significant offline preprocessing. Building on our re- cent work, we first introduce a model-free approach to meta-level control based on reinforcement learn- ing. We then present a meta-level control technique that uses temporal difference learning. Finally, we show empirically that our approach is effective on a common benchmark in meta-level control.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZlta18,

title = {Policy Networks for Reasoning in Long-Term Autonomy},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZlta18.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {AAAI Fall Symposium on Reasoning and Learning in Real-World Systems for Long-Term Autonomy (LTA)},

address = {Arlington, Virginia},

abstract = {Policy networks are graphical models that integrate decision-making models. They allow for multiple Markov decision processes (MDPs) that describe distinct focused aspects of a domain to work in harmony to solve a large-scale problem. This paper presents the formalization of policy networks and their use in modeling reasoning tasks necessary for scalable long-term autonomy. We prove that policy networks generalize a wide array of previous models, such as options and constrained MDPs, which can be equivalently viewed as the integration of multiple models. To illustrate the approach, we apply policy networks to the challenging real world domain of robotic home health care. We demonstrate the benefits of policy networks on a real robot and show how they facilitate scalable integration of multiple decision-making models.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WKSZaaai17,

title = {Robust Optimization for Tree-Structured Stochastic Network Design},

author = {Xiaojian Wu and Akshat Kumar and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WKSZaaai17.pdf},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 31st Conference on Artificial Intelligence (AAAI)},

pages = {4545--4551},

address = {San Francisco, California},

abstract = {Stochastic network design is a general framework for optimizing network connectivity. It has several applications in computational sustainability including spatial conservation planning, pre-disaster network preparation, and river net- work optimization. A common assumption in previous work has been made that network parameters (e.g., probability of species colonization) are precisely known, which is unrealistic in real-world settings. We therefore address the robust river network design problem where the goal is to optimize river connectivity for fish movement by removing barriers. We assume that fish passability probabilities are known only imprecisely, but are within some interval bounds. We then develop a planning approach that computes the policies with either high robust ratio or low regret. Empirically, our approach scales well to large river networks. We also provide insights into the solutions generated by our robust approach, which has significantly higher robust ratio than the baseline solution with mean parameter estimates.},

note = {Best Paper Award},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FZaaai17,

title = {Integration of Planning with Recognition for Responsive Interaction Using Classical Planners},

author = {Richard G Freedman and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FZaaai17.pdf},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 31st Conference on Artificial Intelligence (AAAI)},

pages = {4581--4588},

address = {San Francisco, California},

abstract = {Interaction between multiple agents requires some form of coordination and a level of mutual awareness. When computers and robots interact with people, they need to recognize human plans and react appropriately. Plan and goal recognition techniques have focused on identifying an agent's task given a sufficiently long action sequence. However, by the time the plan and/or goal are recognized, it may be too late for computing an interactive response. We propose an integration of planning with probabilistic recognition where each method uses intermediate results from the other as a guid- ing heuristic for recognition of the plan/goal in-progress as well as the interactive response. We show that, like the used recognition method, these interaction problems can be com- piled into classical planning problems and solved using off- the-shelf methods. In addition to the methodology, this paper introduces problem categories for different forms of interaction, an evaluation metric for the benefits from the interaction, and extensions to the recognition algorithm that make its intermediate results more practical while the plan is in progress.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PWZaaai17,

title = {Fast SSP Solvers Using Short-Sighted Labeling},

author = {Luis Enrique Pineda and Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PWZaaai17.pdf},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 31st Conference on Artificial Intelligence (AAAI)},

pages = {3629--3635},

address = {San Francisco, California},

abstract = {State-of-the-art methods for solving SSPs often work by limiting planning to restricted regions of the state space. The resulting problems can then be solved quickly, and the pro- cess is repeated during execution when states outside the restricted region are encountered. Typically, these approaches focus on states that are within some distance measure of the start state (e.g., number of actions or probability of being reached). However, these short-sighted approaches make it difficult to propagate information from states that are closer to a goal than to the start state, thus missing opportunities to improve planning. We present an alternative approach in which short-sightedness is used only to determine whether a state should be labeled as solved or not, but otherwise the set of states that can be accounted for during planning is unrestricted. Based on this idea, we propose the FLARES algorithm and show that it performs consistently well on a wide range of benchmark problems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZplanrob17,

title = {Approximating Reachable Belief Points in POMDPs with Applications to Robotic Navigation and Localization},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZplanrob17.pdf},

year  = {2017},

date = {2017-01-01},

booktitle = {ICAPS Workshop on Planning and Robotics (PlanRob)},

address = {Pittsburgh, Pennsylvania},

abstract = {Stochastic network design is a general framework for optimizing network connectivity. It has several applications in computational sustainability including spatial conservation planning, pre-disaster network preparation, and river net- work optimization. A common assumption in previous work has been made that network parameters (e.g., probability of species colonization) are precisely known, which is unrealistic in real-world settings. We therefore address the robust river network design problem where the goal is to optimize river connectivity for fish movement by removing barriers. We assume that fish passability probabilities are known only imprecisely, but are within some interval bounds. We then develop a planning approach that computes the policies with either high robust ratio or low regret. Empirically, our approach scales well to large river networks. We also provide insights into the solutions generated by our robust approach, which has significantly higher robust ratio than the baseline solution with mean parameter estimates.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KPGKZhsdip17,

title = {Equi-Reward Utility Maximizing Design in Stochastic Environments},

author = {Sarah Keren and Luis Enrique Pineda and Avigdor Gal and Erez Karpas and Shlomo Zilberstein},

year  = {2017},

date = {2017-01-01},

booktitle = {ICAPS Workshop on Heuristics and Search for Domain-independent Planning (HSDIP)},

address = {Pittsburgh, Pennsylvania},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZSspark17,

title = {Optimizing Electric Vehicle Charging Through Determinization},

author = {Sandhya Saisubramanian and Shlomo Zilberstein and Prashant Shenoy},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZSspark17.pdf},

year  = {2017},

date = {2017-01-01},

booktitle = {ICAPS Workshop on Scheduling and Planning Applications (SPARK)},

address = {Pittsburgh, Pennsylvania},

abstract = {We propose a determinization based approach to optimize the charging policies of an electric vehicle (EV) operating in a vehicle-to-grid (V2G) setting. By planning when to charge or discharge electricity from the vehicle, the long-term cost of operating the EV can be minimized, while being consistent with the owner's preferences. For an EV operating under price uncertainty caused by the dynamic pricing of electricity, this problem needs to be solved on-the-fly. Therefore, we model this problem as a Stochastic Shortest Path (SSP) problem and employ a determinization technique to solve it. Since it is hard to predict a priori the performance of a determinization method on a given problem, we introduce the notion of Lossless Determinization (LLD) that produces optimal action selection via determinization and present an approach that achieves lossless determinization by adjusting the cost of actions to account for the ignored outcomes. We also present Approximate Lossless Determinization (ALLD)--an effective method for approximating the cost of actions based on state features. We evaluate the performance of ALLD and demonstrate its effectiveness on a range of settings for the electric vehicle charging problem.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FZkeps17,

title = {A PDDL Representation for Contradance Composition},

author = {Richard G Freedman and Shlomo Zilberstein},

year  = {2017},

date = {2017-01-01},

booktitle = {ICAPS Workshop on Knowledge Engineering for Planning and Scheduling (KEPS)},

address = {Pittsburgh, Pennsylvania},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WWZijcai17,

title = {Online Decision Making for Scalable Autonomous Systems},

author = {Kyle Hollins Wray and Stefan J Witwicki and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WWZijcai17.pdf},

doi = {10.24963/ijcai.2017/664},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 26th International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {4768--4774},

abstract = {We present a general formal model called MODIA that can tackle a central challenge for autonomous vehicles (AVs), namely the ability to interact with an unspecified, large number of world entities. In MODIA, a collection of possible decision- problems (DPs), known a priori, are instantiated online and executed as decision-components (DCs), unknown a priori. To combine the individual action recommendations of the DCs into a single action, we propose the lexicographic executor action function (LEAF) mechanism. We analyze the complexity of MODIA and establish LEAF's relation to regret minimization. Finally, we implement MODIA and LEAF using collections of partially observable Markov decision pro- cess (POMDP) DPs, and use them for complex AV intersection decision-making. We evaluate the approach in six scenarios within a realistic vehicle simulator and present its use on an AV prototype.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZCijcai17,

title = {Multi-Agent Planning with Baseline Regret Minimization},

author = {Feng Wu and Shlomo Zilberstein and Xiaoping Chen},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZCijcai17.pdf},

doi = {10.24963/ijcai.2017/63},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 26th International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {444--450},

abstract = {We propose a novel baseline regret minimization algorithm for multi-agent planning problems modeled as finite-horizon decentralized POMDPs. It guarantees to produce a policy that is provably at least as good as a given baseline policy. We also propose an iterative belief generation algorithm to efficiently minimize the baseline regret, which only requires necessary iterations so as to converge to the policy with minimum baseline regret. Experimental results on common benchmark problems confirm the benefits of the algorithm compared with the state-of-the-art approaches.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:ZCZWCicira17,

title = {Toward Effective Soft Robot Control via Reinforcement Learning},

author = {Haochong Zhang and Rongyun Cao and Shlomo Zilberstein and Feng Wu and Xiaoping Chen},

url = {https://doi.org/10.1007/978-3-319-65289-4_17},

doi = {10.1007/978-3-319-65289-4_17},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 10th International Conference on Intelligent Robotics and Applications},

pages = {173--184},

abstract = {A soft robot is a kind of robot that is constructed with soft, deformable and elastic materials. Control of soft robots presents complex modeling and planning challenges. We introduce a new approach to accomplish that, making two key contributions: designing an abstract representation of the state of soft robots, and developing a reinforcement learning method to derive effective control policies. The reinforcement learning process can be trained quickly by ignoring the specific materials and structural properties of the soft robot. We apply the approach to the Honeycomb PneuNets Soft Robot and demonstrate the effectiveness of the training method and its ability to produce good control policies under different conditions.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZiros17,

title = {Approximating reachable belief points in POMDPs},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZiros17.pdf},

doi = {10.1109/IROS.2017.8202146},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {117--122},

address = {Vancouver, BC, Canada},

abstract = {We propose an algorithm called ?-approximation that compresses the non-zero values of beliefs for partially observable Markov decision processes (POMDPs) in order to improve performance and reduce memory usage. Specifically, we approximate individual belief vectors with a fixed bound on the number of non-zero values they may contain. We prove the correctness and a strong error bound when the ?-approximation is used with the point-based value iteration (PBVI) family algorithms. An analysis compares the algorithm on six larger domains, varying the number of non-zero values for the ?-approximation. Results clearly demonstrate that when the algorithm used with PBVI (?-PBVI), we can achieve over an order of magnitude improvement. We ground our claims with a full robotic implementation for simultaneous navigation and localization using POMDPs with ?-PBVI.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KPGKZijcai17,

title = {Equi-Reward Utility Maximizing Design in Stochastic Environments},

author = {Sarah Keren and Luis Enrique Pineda and Avigdor Gal and Erez Karpas and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/KPGKZijcai17.pdf},

doi = {10.24963/ijcai.2017/608},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 26th International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {4353--4360},

address = {Melbourne, Australia},

abstract = {We present the Equi-Reward Utility Maximizing Design (ER-UMD) problem for redesigning stochastic environments to maximize agent performance. ER-UMD fits well contemporary applications that require offline design of environments where robots and humans act and cooperate. To find an optimal modification sequence we present two novel solution techniques: a compilation that embeds design into a planning problem, allowing use of off-the-shelf solvers to find a solution, and a heuristic search in the modifications space, for which we present an admissible heuristic. Evaluation shows the feasibility of the approach using standard benchmarks from the probabilistic planning competition and a benchmark we created for a vacuum cleaning robot setting.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZaaai16,

title = {A POMDP Formulation of Proactive Learning},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZaaai16.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the 30th Conference on Artificial Intelligence (AAAI)},

pages = {3202--3208},

address = {Phoenix, Arizona},

abstract = {We cast the Proactive Learning (PAL) problem--Active Learning (AL) with multiple reluctant, fallible, cost-varying oracles--as a Partially Observable Markov Decision Process (POMDP). The agent selects an oracle at each time step to label a data point while it maintains a belief over the true underlying correctness of its current dataset's labels. The goal is to minimize labeling costs while considering the value of obtaining correct labels, thus maximizing final resultant classifier accuracy. We prove three properties that show our particular formulation leads to a structured and bounded-size set of belief points, enabling strong performance of point-based methods to solve the POMDP. Our method is compared with the original three algorithms proposed by Donmez and Carbonell and a simple baseline. We demonstrate that our approach matches or improves upon the original approach within five different oracle scenarios, each on two datasets. Finally, our algorithm provides a general, well-defined mathematical foundation to build upon.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WSZaaai16,

title = {Optimizing Resilience in Large Scale Networks},

author = {Xiaojian Wu and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WSZaaai16.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the 30th Conference on Artificial Intelligence (AAAI)},

pages = {3922--3928},

address = {Phoenix, Arizona},

abstract = {We propose a decision making framework to optimize the resilience of road networks to natural disasters such as floods. Our model generalizes an existing one for this problem by allowing roads with a broad class of stochastic delay models. We then present a fast algorithm based on the sample average approximation (SAA) method and network design techniques to solve this problem approximately. On a small existing benchmark, our algorithm produces near-optimal solutions and the SAA method converges quickly with a small number of samples.We then apply our algorithm to a large real-world problem to optimize the resilience of a road network to failures of stream crossing structures to minimize travel times of emergency medical service vehicles. On medium-sized networks, our algorithm obtains solutions of comparable quality to a greedy baseline method but is 30--60 times faster. Our algorithm is the only existing algorithm that can scale to the full network, which has many thousands of edges.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WPZaamas16,

title = {Hierarchical Approach to Transfer of Control in Semi-Autonomous Systems (Extended Abstract)},

author = {Kyle Hollins Wray and Luis Enrique Pineda and Shlomo Zilberstein},

url = {http://dl.acm.org/citation.cfm?id=2937122},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the 15th International Conference on Autonomous Agents and Multiagent Systems (AAMAS)},

pages = {1285--1286},

address = {Singapore},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KMZicaps16,

title = {Dual Formulations for Optimizing Dec-POMDP Controllers},

author = {Akshat Kumar and Hala Mostafa and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/KMZicaps16.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the 26th International Conference on Automated Planning and Scheduling (ICAPS)},

pages = {202--210},

address = {London, UK},

abstract = {Decentralized POMDP is an expressive model for multiagent planning. Finite-state controllers (FSCs)--often used to represent policies for infinite-horizon problems---offer a compact, simple-to-execute policy representation. We exploit novel connections between optimizing decentralized FSCs and the dual linear program for MDPs. Consequently, we describe a dual mixed integer linear program (MIP) for optimizing deterministic FSCs. We exploit the Dec-POMDP structure to devise a compact MIP and formulate constraints that result in policies executable in partially-observable decentralized settings. We show analytically that the dual formulation can also be exploited within the expectation maximization (EM) framework to optimize stochastic FSCs. The resulting EM algorithm can be implemented by solving a sequence of linear programs, without requiring expensive message passing over the Dec-POMDP DBN. We also present an efficient technique for policy improvement based on a weighted entropy measure. Compared with state-of-the-art FSC methods, our approach offers over an order-of-magnitude speedup, while producing similar or better solutions.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WPZijcai16,

title = {Hierarchical Approach to Transfer of Control in Semi-Autonomous Systems},

author = {Kyle Hollins Wray and Luis Enrique Pineda and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WPZijcai16.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the 25th International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {517--523},

address = {New York, NY},

abstract = {Semi-Autonomous Systems (SAS) encapsulate a stochastic decision process explicitly controlled by both an agent and a human, in order to leverage the distinct capabilities of each actor. Planning in SAS must address the challenge of transferring control quickly, safely, and smoothly back-and-forth between the agent and the human. We formally define SAS and the requirements to guarantee that the controlling entities are always able to act competently. We then consider applying the model to Semi-Autonomous VEhicles (SAVE), using a hierarchical approach in which micro-level transfer-of-control actions are governed by a high-fidelity POMDP model. Macro-level path planning in our hierarchical approach is performed by solving a Stochastic Shortest Path (SSP) problem. We analyze the integrated model and show that it provides the required guarantees. Finally, we test the SAVE model using real-world road data from Open Street Map (OSM) within 10 cities, showing the benefits of the collaboration between the agent and human.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PWZijcai16ws,

title = {Fast SSP Solvers Using Short-Sighted Labeling},

author = {Luis Enrique Pineda and Kyle Hollins Wray and Shlomo Zilberstein},

year  = {2016},

date = {2016-01-01},

booktitle = {IJCAI Workshop on Goal Reasoning},

address = {New York, NY},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FZijcai16ws2,

title = {Using Metadata to Automate Interpretations of Unsupervised Learning-Derived Clusters},

author = {Richard G Freedman and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FZijcai16ws2.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {IJCAI Workshop on Human is More Than a Labeler},

address = {New York, NY},

abstract = {Unsupervised machine learning methods are useful for identifying clusters of similar inputs with re- spect to some criteria and giving the inputs within each cluster the same label. However, the results of many such methods rely on parameter choices that can alter the derived classification labels for each input. Verification methods for determining the quality of clusters often rely on human intuition, but this is not always an easy task depending on the format of the inputs and finding the correct relationship that the algorithm used.We present an approach to assist human verification of the unsupervised learning algorithms' classification choices through the use of metadata describing the inputs to be clustered. When the metadata measures the relevance of each input to human-interpretable features, we show how a similar measurement of relevance to human-interpretable features can be de- rived to describe the unsupervised learning algorithm's choices of clusters. An example demonstrating how it evaluates previous work with activity recognition via topic models is provided in addition to propositions of other uses for the metadata.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WRGZiros16,

title = {Log-Space Harmonic Function Path Planning},

author = {Kyle Hollins Wray and Dirk Ruiken and Roderic A Grupen and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WRGZiros16.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS)},

pages = {1511--1516},

address = {Daejeon, South Korea},

abstract = {We propose a log-space solution for robotic path planning with harmonic functions that solves the long-standing numerical precision problem. We prove that this algorithm: (1) performs the correct computations in log-space, (2) returns the true equivalent path using the log-space mapping, and (3) has a strong error bound given its convergence criterion. We evaluate the algorithm on 7 problem domains. A Graphics Processing Unit (GPU) implementation is also shown to greatly improve performance. We also provide an open source library entitled epic with extensive ROS support and demonstrate this method on a real humanoid robot: the uBot-6. Experiments demonstrate that the log-space solution rapidly produces smooth obstacle-avoiding trajectories, and supports planning in exponentially larger real-world robotic applications.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FZfall16,

title = {Safety in AI-HRI: Challenges Complementing User Experience Quality},

author = {Richard G Freedman and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FZfall16.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {AAAI Fall Symposium on Artificial Intelligence and Human-Robot Interaction (AI-HRI)},

address = {Arlington, Virginia},

abstract = {Contemporary research in human-robot interaction (HRI) predominantly focuses on the user's experience while con- trolling a robot. However, with the increased deployment of artificial intelligence (AI) techniques, robots are quickly be- coming more autonomous in both academic and industrial experimental settings. In addition to improving the user's inter- active experience with AI-operated robots through personalization, dialogue, emotions, and dynamic behavior, there is also a growing need to consider the safety of the interaction. AI may not account for the user's less likely responses, making it possible for an unaware user to be injured by the robot if they have a collision. Issues of trust and acceptance may also come into play if users cannot always understand the robot's thought process, creating a potential for emotional harm. We identify challenges that will need to be addressed in safe AI-HRI and provide an overview of approaches to consider for them, many stemming from the contemporary research.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZMaaai15,

title = {Multi-Objective MDPs with Conditional Lexicographic Reward Preferences},

author = {Kyle Hollins Wray and Shlomo Zilberstein and Abdel-Illah Mouaddib},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZMaaai15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 29th Conference on Artificial Intelligence (AAAI)},

pages = {3418--3424},

address = {Austin, Texas},

abstract = {Sequential decision problems that involve multiple objectives are prevalent. Consider for example a driver of a semi-autonomous car who may want to optimize competing objectives such as travel time and the effort associated with manual driving. We introduce a rich model called Lexicographic MDP (LMDP) and a corresponding planning algorithm called LVI that generalize previous work by allowing for conditional lexicographic preferences with slack. We analyze the convergence characteristics of LVI and establish its game theoretic properties. The performance of LVI in practice is tested within a realistic benchmark problem in the domain of semi-autonomous driving. Finally, we demonstrate how GPU-based optimization can improve the scalability of LVI and other value iteration algorithms for MDPs.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:Zaaai15,

title = {Building Strong Semi-Autonomous Systems},

author = {Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/Zaaai15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 29th Conference on Artificial Intelligence (AAAI)},

pages = {4088--4092},

address = {Austin, Texas},

abstract = {The vision of populating the world with autonomous systems that reduce human labor and improve safety is gradually becoming a reality. Autonomous systems have changed the way space exploration is conducted and are beginning to transform everyday life with a range of household products. In many areas, however, there are considerable barriers to the deployment of fully autonomous systems. We refer to systems that require some degree of human intervention in order to complete a task as semi-autonomous systems. We examine the broad rationale for semi-autonomy and define basic properties of such systems. Accounting for the human in the loop presents a considerable challenge for current planning techniques. We examine various design choices in the development of semi-autonomous systems and their implications on planning and execution. Finally, we discuss fruitful research directions for advancing the science of semi-autonomy.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:SZGARaaai15,

title = {Tractability of Planning with Loops},

author = {Siddharth Srivastava and Shlomo Zilberstein and Abhishek Gupta and Pieter Abbeel and Stuart J Russell},

url = {http://rbr.cs.umass.edu/shlomo/papers/SZGARaaai15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 29th Conference on Artificial Intelligence (AAAI)},

pages = {3393--3401},

address = {Austin, Texas},

abstract = {We create a unified framework for analyzing and synthesizing plans with loops for solving problems with non-deterministic numeric effects and a limited form of partial observability. Three different action models -- with deterministic, qualitative non-deterministic and Boolean non-deterministic semantics -- are handled using a single abstract representation. We establish the conditions under which the correctness and termination of solutions, represented as abstract policies, can be verified. We also examine the feasibility of learning abstract policies from examples. We demonstrate our techniques on several planning problems and show that they apply to challenging real-world tasks such as doing the laundry with a PR2 robot. These results resolve a number of open questions about planning with loops and facilitate the development of new algorithms and applications.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:JFFCZGiui15,

title = {Learning Therapy Strategies from Demonstration Using Latent Dirichlet Allocation},

author = {Hee-Tae Jung and Richard G Freedman and Tammie Foster and Yu-Kyong Choe and Shlomo Zilberstein and Roderic A Grupen},

url = {http://rbr.cs.umass.edu/shlomo/papers/JFFCZGiui15.pdf},

doi = {10.1145/2678025.2701403},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 20th ACM Conference on Intelligent User Interfaces (IUI)},

pages = {432--436},

address = {Atlanta, Georgia},

abstract = {The use of robots in stroke rehabilitation has become a popular trend in rehabilitation robotics. However, despite the acknowledged value of customized service for individual patients, research on programming adaptive therapy for individual patients has received little attention. The goal of the current study is to model tele-therapy sessions in the form of a generative process for autonomous therapy that approximate the demonstrations of the therapist. The resulting autonomous programs for therapy may imitate the strategy that the therapist might have employed and reinforce therapeutic exercises between tele-therapy sessions. We propose to encode the therapist's decision criteria in terms of the patient's motor performance features. Specifically, in this work, we apply Latent Dirichlet Allocation on the batch data collected during tele-therapy sessions between a single stroke patient and a single therapist. Using the resulting models, the therapeutic exercise targets are generated and are verified with the same therapist who generated the data.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KZicaps15,

title = {History-Based Controller Design and Optimization for Partially Observable MDPs},

author = {Akshat Kumar and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/KZicaps15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 25th International Conference on Automated Planning and Scheduling (ICAPS)},

pages = {156--164},

address = {Jerusalem, Israel},

abstract = {Partially observable MDPs provide an elegant framework for sequential decision making. Finite-state controllers (FSCs) are often used to represent policies for infinite-horizon problems as they offer a compact representation, simple-to-execute plans, and adjustable tradeoff between computational complexity and policy size. We develop novel connections between optimizing FSCs for POMDPs and the dual linear program for MDPs. Building on that, we present a dual mixed integer linear program (MIP) for optimizing FSCs. To assign well-defined meaning to FSC nodes as well as aid in policy search, we show how to associate history-based features with each FSC node. Using this representation, we address another challenging problem, that of iteratively deciding which nodes to add to FSC to get a better policy. Using an efficient off-the-shelf MIP solver, we show that this new approach can find compact near-optimal FSCs for several large benchmark domains, and is competitive with previous best approaches.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:MJZplanrob15,

title = {Handling Advice in MDPs for Semi-Autonomous Systems},

author = {Abdel-Illah Mouaddib and Laurent Jeanpierre and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/MJZplanrob15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {ICAPS Workshop on Planning and Robotics (PlanRob)},

address = {Jerusalem, Israel},

abstract = {This paper proposes an effective new model for decision making in situations where full autonomy is not feasible due to the inability to fully model and reason about the domain. To overcome this limitation, we consider a human operator who can supervise the system and guide its operation by providing high-level advice. We define a rich representation for advice and describe an effective algorithm for generating a new policy that conforms to the given advice. Advice is designed to improve the efficiency and safety of the system by imposing constraints on state visitation (either encouraging or discouraging the system to visit certain states). Coupled with the standard reward maximization criterion for MDPs, advice poses a complex multi-criteria decision problem. We present and analyze an effective algorithm for optimizing the policy in the presence of advice.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KZijcai15,

title = {Multi-Objective POMDPs with Lexicographic Reward Preferences},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/KZijcai15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 24th International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {1719--1725},

address = {Buenos Aires, Argentina},

abstract = {We propose a model, Lexicographic Partially Observable Markov Decision Process (LPOMDP), which extends POMDPs with lexicographic preferences over multiple value functions. It allows for slack--slightly less-than-optimal values--for higher-priority preferences to facilitate improvement in lower-priority value functions. Many real life situations are naturally captured by LPOMDPs with slack. We consider a semi-autonomous driving scenario in which time spent on the road is minimized, while maximizing time spent driving autonomously. We propose two solutions to LPOMDPs--Lexicographic Value Iteration (LVI) and Lexicographic Point-Based Value Iteration (LPBVI), establishing convergence results and correctness within strong slack bounds. We test the algorithms using real-world road data provided by Open Street Map (OSM) within 10 major cities. Finally, we present GPU-based optimizations for point-based solvers, demonstrating that their application enables us to quickly solve vastly larger LPOMDPs and other variations of POMDPs.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WSZijcai15,

title = {Fast Combinatorial Algorithm for Optimizing the Spread of Cascades},

author = {Xiaojian Wu and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WSZijcai15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the 24th International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {2655--2661},

address = {Buenos Aires, Argentina},

abstract = {We address a spatial conservation planning problem in which the planner purchases a budget-constrained set of land parcels in order to maximize the expected spread of a population of an endangered species. Existing techniques based on the sample average approximation scheme and standard integer programming methods have high complexity and limited scalability. We propose a fast combinatorial optimization algorithm using Lagrangian relaxation and primal-dual techniques to solve the problem approximately. The algorithm provides a new way to address a range of conservation planning and scheduling problems. On the Red-cockaded Woodpecker data, our algorithm produces near optimal solutions and runs significantly faster than a standard mixed integer program solver. Compared with a greedy baseline, the solution quality is comparable or better, but our algorithm is 10--30 times faster. On synthetic problems that do not exhibit submodularity, our algorithm significantly outperforms the greedy baseline.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:JFTWZGCicorr15,

title = {Adaptive Therapy Strategies: Efficacy and Learning Framework},

author = {Hee-Tae Jung and Richard G Freedman and Takeshi Takahashi and Jay Ming Wong and Shlomo Zilberstein and Roderic A Grupen and Yu-kyong Choe},

url = {http://rbr.cs.umass.edu/shlomo/papers/JFTWZGCicorr15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the IEEE/RAS-EMBS International Conference on Rehabilitation Robotics},

address = {Singapore},

abstract = {This paper considers a data-driven framework to model target selection strategies using runtime kinematic parameters of individual patients. These models can be used to select new exercise targets that conform with the decision criteria of the therapist. We present the results from a single-subject case study with a manually written target selection function. Motivated by promising results, we propose a framework to learning customized/adaptive therapy models for individual patients. Through the data collected from a normally functioning adult, we demonstrate that it is feasible to model varying strategies from the demonstration of target selection.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FZromanW15,

title = {Automated Interpretations of Unsupervised Learning-Derived Clusters for Activity Recognition},

author = {Richard G Freedman and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FZromanW15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {Ro-Man Workshop on Learning for Human-Robot Collaboration},

address = {Kobe, Japan},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PTJZGhuman15,

title = {Continual Planning for Search and Rescue Robots},

author = {Luis Pineda and Takeshi Takahashi and Hee-Tae Jung and Shlomo Zilberstein and Roderic A Grupen},

url = {http://rbr.cs.umass.edu/shlomo/papers/PTJZGhuman15.pdf},

doi = {10.1109/HUMANOIDS.2015.7363542},

year  = {2015},

date = {2015-01-01},

booktitle = {Proceedings of the IEEE-RAS 15th International Conference on Humanoid Robots},

pages = {243--248},

address = {Seoul, Korea},

abstract = {The deployment of robots for emergency response tasks such as search and rescue is a promising application of robotics with growing importance. Given the perilous nature of these tasks, autonomous robot operation is highly desirable in order to reduce the risk imposed on the human rescue team. While much work has been done on creating robotic systems that can be deployed for search and rescue, limited work has been devoted to devise efficient real-time automated planning algorithms for these tasks. In this work, we present REDHI, an efficient algorithm for solving probabilistic models of complex problems such as search and rescue. We evaluate our algorithm on the search and rescue problem using both an abstract domain representation and a semi-realistic simulator of an existing robot system. The results show that REDHI can obtain near optimal performance with negligible planning time.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FJZfall15,

title = {Temporal and Object Relations in Unsupervised Plan and Activity Recognition},

author = {Richard G Freedman and Hee-Tae Jung and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FJZfall15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {AAAI Fall Symposium on Artificial Intelligence and Human-Robot Interaction (AI-HRI)},

address = {Arlington, Virginia},

abstract = {We consider ways to improve the performance of unsupervised plan and activity recognition techniques by considering temporal and object relations in addition to postural data. Temporal relationships can help recognize activities with cyclic structure and are often implicit because plans have degrees of ordering actions. Relations with objects can help disambiguate observed activities that otherwise share a user's posture and position. We develop and investigate graphical models that extend the popular latent Dirichlet allocation approach with temporal and object relations, examine the relative performance and runtime trade-offs using a standard dataset, and consider the cost/benefit trade-offs these extensions offer in the context of human-robot and human- computer interaction.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PWZfall15,

title = {Revisiting Multi-Objective MDPs with Relaxed Lexicographic Preferences},

author = {Luis Pineda and Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PWZfall15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {AAAI Fall Symposium on Sequential Decision Making for Intelligent Agents (SDMIA)},

address = {Arlington, Virginia},

abstract = {We consider stochastic planning problems that involve multiple objectives such as minimizing task completion time and energy consumption. These problems can be modeled as multi-objective Markov decision processes (MOMDPs), an extension of the widely-used MDP model to handle problems involving multiple value functions. We focus on a subclass of MOMDPs in which the objectives have a relaxed lexicographic structure, allowing an agent to seek improvement in a lower-priority objective when the impact on a higher- priority objective is within some small given tolerance. We examine the relationship between this class of problems and constrained MDPs, showing that the latter offer an alternative solution method with strong guarantees. We show empirically that a recently introduced algorithm for MOMDPs may not offer the same strong guarantees, but it does perform well in practice.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZfall15,

title = {A Parallel Point-Based POMDP Algorithm Leveraging GPUs},

author = {Kyle Hollins Wray and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZfall15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {AAAI Fall Symposium on Sequential Decision Making for Intelligent Agents (SDMIA)},

address = {Arlington, Virginia},

abstract = {We parallelize the Point-Based Value Iteration (PBVI) algorithm, which approximates the solution to Partially Observable Markov Decision Processes (POMDPs), using a Graph- ics Processing Unit (GPU). We detail additional optimizations, such as leveraging the bounded size of non-zero values over all belief point vectors, usable by serial and parallel algorithms. We compare serial (CPU) and parallel (GPU) implementations on 10 distinct problem domains, and demonstrate that our approach provides an order of magnitude improvement.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WSZnipsW15,

title = {Efficient Algorithms to Optimize Diffusion Processes under the Independent Cascade Model},

author = {Xiaojian Wu and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WSZnipsW15.pdf},

year  = {2015},

date = {2015-01-01},

booktitle = {NIPS Workshop on Networks in the Social and Information Sciences},

address = {Montreal, Quebec},

abstract = {We study scalable algorithms to optimize diffusion processes under the Independent Cascade model. We consider a broad class of intervention actions, including selecting sources, raising the probability that the diffusion propagates from one node to another and changing the topology of networks to facilitate the diffusion. Optimizing the selection of such actions with a limited budget tends to be NP- hard and is neither submodular nor supermodular. We provide scalable algorithms for three different problem settings that range in terms of the strength of the assumptions we make about the model. The algorithms are very efficient (faster than a baseline greedy algorithm), producing high-quality solutions in several diffusion maximization problems in the area of computational sustainability and in some cases also have provable approximation guarantees. These techniques offer promising results that may be applied to diffusion optimization problems in social and information networks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NYLZZaamas14,

title = {Decentralized Multi-Agent Reinforcement Learning in Average-Reward Dynamic DCOPs},

author = {Duc Thien Nguyen and William Yeoh and Hoong Chuin Lau and Shlomo Zilberstein and Chongjie Zhang},

url = {http://rbr.cs.umass.edu/shlomo/papers/NYLZZaamas14.pdf},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 13th International Conference on Autonomous Agents and Multiagent Systems (AAMAS)},

pages = {1341--1342},

address = {Paris, France},

abstract = {Researchers have introduced the Dynamic Distributed Constraint Optimization Problem (Dynamic DCOP) formulation to model dynamically changing multi-agent coordination problems, where a dynamic DCOP is a sequence of (static canonical) DCOPs, each partially different from the DCOP preceding it. Existing work typically assumes that the problem in each time step is decoupled from the problems in other time steps, which might not hold in some applications. In this paper, we introduce a new model, called Markovian Dynamic DCOPs (MD-DCOPs), where a DCOP is a function of the value assignments in the preceding DCOP. We also introduce a distributed reinforcement learning algorithm that balances exploration and exploitation to solve MD-DCOPs in an online manner.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:FJZicaps14,

title = {Plan and Activity Recognition from a Topic Modeling Perspective},

author = {Richard G Freedman and Hee-Tae Jung and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/FJZicaps14.pdf},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 24thInternational Conference on Automated Planning and Scheduling (ICAPS)},

pages = {360--364},

address = {Portsmouth, New Hampshire},

abstract = {We examine new ways to perform plan recognition (PR) using natural language processing (NLP) techniques. PR often focuses on the structural relationships between consecutive observations and ordered activities that comprise plans. However, NLP commonly treats text as a bag-of-words, omitting such structural relationships and using topic models to break down the distribution of concepts discussed in documents. In this paper, we examine an analogous treatment of plans as distributions of activities. We explore the application of Latent Dirichlet Allocation topic models to human skeletal data of plan execution traces obtained from a RGB-D sensor. This investigation focuses on representing the data as text and interpreting learned activities as a form of activity recognition (AR). Additionally, we explain how the system may perform PR. The initial empirical results suggest that such NLP methods can be useful in complex PR and AR tasks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PZicaps14,

title = {Planning Under Uncertainty Using Reduced Models: Revisiting Determinization},

author = {Luis Pineda and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/PZicaps14.pdf},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 24th International Conference on Automated Planning and Scheduling (ICAPS)},

pages = {217--225},

address = {Portsmouth, New Hampshire},

abstract = {We introduce a family of MDP reduced models characterized by two parameters: the maximum number of primary outcomes per action that are fully accounted for and the maximum number of occurrences of the remaining exceptional outcomes that are planned for in advance. Reduced models can be solved much faster using heuristic search algorithms such as LAO*, benefiting from the dramatic reduction in the number of reachable states. A commonly used determinization approach is a special case of this family of reductions, with one primary outcome per action and zero exceptional outcomes per plan. We present a framework to compute the benefits of planning with reduced models, relying on online planning when the number of exceptional outcomes exceeds the bound. Using this framework, we compare the performance of various reduced models and consider the challenge of generating good ones automatically. We show that each one of the dimensions--allowing more than one primary outcome or planning for some limited number of exceptions--could improve performance relative to standard determinization. The results place recent work on determinization in a broader context and lay the foundation for efficient and systematic exploration of the space of MDP model reductions.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:NYLZZaaai14,

title = {Decentralized Multi-Agent Reinforcement Learning in Average-Reward Dynamic DCOPs},

author = {Duc Thien Nguyen and William Yeoh and Hoong Chuin Lau and Shlomo Zilberstein and Chongjie Zhang},

url = {http://rbr.cs.umass.edu/shlomo/papers/NYLZZaaai14.pdf},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 28th Conference on Artificial Intelligence (AAAI)},

pages = {1447--1455},

address = {Quebec City, Canada},

abstract = {Researchers have introduced the Dynamic Distributed Constraint Optimization Problem (Dynamic DCOP) formulation to model dynamically changing multi-agent coordination problems, where a dynamic DCOP is a sequence of (static canonical) DCOPs, each partially different from the DCOP preceding it. Existing work typically assumes that the problem in each time step is decoupled from the problems in other time steps, which might not hold in some applications. Therefore, in this paper, we make the following contributions: (i) We introduce a new model, called Markovian Dynamic DCOPs (MD-DCOPs), where the DCOP in the next time step is a function of the value assignments in the current time step; (ii) We introduce two distributed reinforcement learning algorithms, the Distributed RVI Q-learning algorithm and the Distributed R-learning algorithm, that balance exploration and exploitation to solve MD-DCOPs in an online manner; and (iii) We empirically evaluate them against an existing multiarm bandit DCOP algorithm on dynamic DCOPs.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WSZaaai14,

title = {Rounded Dynamic Programming for Tree-Structured Stochastic Network Design},

author = {Xiaojian Wu and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WSZaaai14.pdf},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 28th Conference on Artificial Intelligence (AAAI)},

pages = {479--485},

address = {Quebec City, Canada},

abstract = {We develop a fast approximation algorithm called rounded dynamic programming (RDP) for stochastic network design problems on directed trees. The underlying model describes phenomena that spread away from the root of a tree, for example, the spread of influence in a hierarchical organization or fish in a river network. Actions can be taken to intervene in the network---for some cost---to increase the probability of propagation along an edge. Our algorithm selects a set of actions to maximize the overall spread in the network under a limited budget. We prove that the algorithm is a fully polynomial-time approximation scheme (FPTAS), that is, it finds (1??)-optimal solutions in time polynomial in the input size and 1/?. We apply the algorithm to the problem of allocating funds efficiently to remove barriers in a river network so fish can reach greater portions of their native range. Our experiments show that the algorithm is able to produce near-optimal solutions much faster than an existing technique.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WSZnips14,

title = {Stochastic Network Design in Bidirected Trees},

author = {Xiaojian Wu and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WSZnips14.pdf},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 28th Neural Information Processing Systems Conference (NIPS)},

pages = {882--890},

address = {Montreal, Canada},

abstract = {We investigate the problem of stochastic network design in bidirected trees. In this problem, an underlying phenomenon (e.g., a behavior, rumor, or disease) starts at multiple sources in a tree and spreads in both directions along its edges. Actions can be taken to increase the probability of propagation on edges, and the goal is to maximize the total amount of spread away from all sources. Our main result is a rounded dynamic programming approach that leads to a fully polynomial-time approximation scheme (FPTAS), that is, an algorithm that can find (1??)-optimal solutions for any problem instance in time polynomial in the input size and 1/?. Our algorithm outperforms competing approaches on a motivating problem from computational sustainability to remove barriers in river networks to restore the health of aquatic ecosystems.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:BSZaaai13,

title = {Qualitative Planning under Partial Observability in Multi-Agent Domains},

author = {Ronen I Brafman and Guy Shani and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/BSZaaai13.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {Proceedings of the 27th Conference on Artificial Intelligence (AAAI)},

pages = {130--137},

address = {Bellevue, Washington},

abstract = {Decentralized POMDPs (Dec-POMDPs) provide a rich, attractive model for planning under uncertainty and partial observability in cooperative multi-agent domains with a growing body of research. In this paper we formulate a qualitative, propositional model for multi-agent planning under uncertainty with partial observability, which we call Qualitative Dec-POMDP (QDec-POMDP). We show that the worst-case complexity of planning in QDec-POMDPs is similar to that of Dec-POMDPs. Still, because the model is more "classical" in nature, it is more compact and easier to specify. Furthermore, it eases the adaptation of methods used in classical and contingent planning to solve problems that challenge current Dec-POMDPs solvers. In particular, in this paper we describe a method based on compilation to classical planning, which handles multi-agent planning problems significantly larger than those handled by current Dec-POMDP algorithms.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:PLZGijcai13,

title = {Fault-Tolerant Planning Under Uncertainty},

author = {Luis Pineda and Yi Lu and Shlomo Zilberstein and Claudia V Goldman},

url = {http://rbr.cs.umass.edu/shlomo/papers/PLZGijcai13.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {Proceedings of the 23rd International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {2350--2356},

address = {Beijing, China},

abstract = {A fault represents some erroneous operation of a system that could result from an action selection error or some abnormal condition. We formally define error models that characterize the likelihood of various faults and consider the problem of fault-tolerant planning, which optimizes performance given an error model. We show that factoring the possibility of errors significantly degrades the performance of stochastic planning algorithms such as LAO*, because the number of reachable states grows dramatically. We introduce an approach to plan for a bounded number of faults and analyze its theoretical properties. When combined with a continual planning paradigm, the k-fault-tolerant planning method can produce near-optimal performance, even when the number of faults exceeds the bound. Empirical results in two challenging domains confirm the effectiveness of the approach in handling different types of runtime errors.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WZJijcai13,

title = {Monte-Carlo Expectation Maximization for Decentralized POMDPs},

author = {Feng Wu and Shlomo Zilberstein and Nicholas R Jennings},

url = {http://rbr.cs.umass.edu/shlomo/papers/WZJijcai13.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {Proceedings of the 23rd International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {397--403},

address = {Beijing, China},

abstract = {We address two significant drawbacks of state-of-the-art solvers of decentralized POMDPs (DEC-POMDPs): the reliance on complete knowledge of the model and limited scalability as the complexity of the domain grows. We extend a recently proposed approach for solving DEC-POMDPs via a reduction to the maximum likelihood problem, which in turn can be solved using EM. We introduce a model-free version of this approach that employs Monte-Carlo EM (MCEM). While a naive implementation of MCEM is inadequate in multi-agent settings, we introduce several improvements in sampling that produce high-quality results on a variety of DEC-POMDP benchmarks, including large problems with thousands of agents.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WKSZijcai13,

title = {Parameter Learning for Latent Network Diffusion},

author = {Xiaojian Wu and Akshat Kumar and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WKSZijcai13.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {Proceedings of the 23rd International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {2923--2930},

address = {Beijing, China},

abstract = {Diffusion processes in networks are increasingly used to model dynamic phenomena such as the spread of information, wildlife, or social influence. Our work addresses the problem of learning the underlying parameters that govern such a diffusion process by observing the time at which nodes become active. A key advantage of our approach is that, unlike previous work, it can tolerate missing observations for some nodes in the diffusion process. Having incomplete observations is characteristic of offline networks used to model the spread of wildlife. We develop an EM algorithm to address parameter learning in such settings. Since both the E and M steps are computationally challenging, we employ a number of optimization methods such as nonlinear and difference-of-convex programming to address these challenges. Evaluation of the approach on the Red-cockaded Woodpecker conservation problem shows that it is highly robust and accurately learns parameters in various settings, even with more than 80% missing data.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:YKZijcai13,

title = {Automated Generation of Interaction Graphs for Value-Factored Dec-POMDPs},

author = {William Yeoh and Akshat Kumar and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/YKZijcai13.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {Proceedings of the 23rd International Joint Conference on Artificial Intelligence (IJCAI)},

pages = {411--417},

address = {Beijing, China},

abstract = {The Decentralized Partially Observable Markov Decision Process (Dec-POMDP) is a powerful model for multiagent planning under uncertainty, but its applicability is hindered by its high complexity -- solving Dec-POMDPs optimally is NEXP-hard. Recently, Kumar et al. introduced the Value Factorization (VF) framework, which exploits decomposable value functions that can be factored into subfunctions. This framework has been shown to be a generalization of several models that leverage sparse agent interactions such as TI-Dec-MDPs, ND-POMDPs and TD-POMDPs. Existing algorithms for these models assume that the interaction graph of the problem is given. In this paper, we introduce three algorithms to automatically generate interaction graphs for models within the VF framework and establish lower and upper bounds on the expected reward of an optimal joint policy. We illustrate experimentally the benefits of these techniques for sensor placement in a decentralized tracking application.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:WSZnips13ws,

title = {Stochastic Network Design for River Networks},

author = {Xiaojian Wu and Daniel Sheldon and Shlomo Zilberstein},

url = {http://rbr.cs.umass.edu/shlomo/papers/WSZnips13ws.pdf},

year  = {2013},

date = {2013-01-01},

booktitle = {NIPS Workshop on Machine Learning for Sustainability},

address = {Lake Tahoe, Nevada},

abstract = {Stochastic network design techniques can be used effectively to solve a wide range of planning problems in ecological sustainability. We propose a novel approximate algorithm based on the sample average approximation (SAA) and mixed integer programming (MIP) to efficiently address the problem of using a limited budget to remove instream barriers, which prevent fish from accessing their natural habitat. In comparison with a dynamic programming (DP) benchmark algorithm, the advantage of our algorithm is the ability to produce a near optimal solution much faster, particularly when the budget is large and the DP based algorithm becomes intractable. Furthermore, while the DP based algorithm can only solve tree-structured stream networks, our algorithm is applicable to networks with a more general directed acyclic graph structure.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}

@conference{SZ:KZTaistats12,

title = {Message-Passing Algorithms for MAP Estimation Using DC Programming},

author = {Akshat Kumar and Shlomo Zilberstein and Marc Toussaint},

url = {http://rbr.cs.umass.edu/shlomo/papers/KZTaistats12.pdf},

year  = {2012},

date = {2012-01-01},

booktitle = {Proceedings of the 15th International Conference on Artificial Intelligence and Statistics (AISTATS)},

pages = {656--664},

address = {La Palma, Canary Islands},

abstract = {We address the problem of finding the most likely assignment or MAP estimation in a Markov random field. We analyze the linear programming formulation of MAP through the lens of difference of convex functions (DC) programming, and use the concave-convex procedure (CCCP) to develop efficient message-passing solvers. The resulting algorithms are guaranteed to converge to a global optimum of the well-studied local polytope, an outer bound on the MAP marginal polytope. To tighten the outer bound, we show how to combine it with the mean-field based inner bound and, again, solve it using CCCP. We also identify a useful relationship between the DC formulations and some recently proposed algorithms based on Bregman divergence. Experimentally, this hybrid approach produces optimal solutions for a range of hard OR problems and nearoptimal solutions for standard benchmarks.},

keywords = {},

pubstate = {published},

tppubtype = {conference}

}