OpenStructure
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
python_tokenizer.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // This file is part of the OpenStructure project <www.openstructure.org>
3 //
4 // Copyright (C) 2008-2011 by the OpenStructure authors
5 //
6 // This library is free software; you can redistribute it and/or modify it under
7 // the terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 3.0 of the License, or (at your option)
9 // any later version.
10 // This library is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
12 // FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
13 // details.
14 //
15 // You should have received a copy of the GNU Lesser General Public License
16 // along with this library; if not, write to the Free Software Foundation, Inc.,
17 // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 //------------------------------------------------------------------------------
19 #ifndef PYTHON_TOKENIZER_HH
20 #define PYTHON_TOKENIZER_HH
21 
22 /*
23  Author: Marco Biasini
24 */
25 
26 
27 #include <ost/gui/module_config.hh>
28 #include <QString>
29 
30 namespace ost { namespace gui {
31 
33 {
34  size_t location, length;
35 
36  Range(size_t loc, size_t len):
37  location(loc),
38  length(len)
39  {
40  }
41 
42  Range():
43  location(-1),
44  length(0)
45  {
46  }
47 
48  size_t End() const
49  {
50  return location+length;
51  }
52 };
53 
55 {
56 public:
57  enum Type {
66  END
67  };
68  PythonToken();
69  PythonToken(Type type,const Range& range, const QString& value);
70  Type GetType() const;
71  QString GetTypeAsString() const;
72  const QString& GetValue() const;
73  const Range& GetRange() const;
74 
75 private:
76  Type type_;
77  Range range_;
78  QString value_;
79 };
80 
82 {
83 public:
85  PythonTokenizer(const QString& command, int string_state=0);
86  void SetCommand(const QString& command);
87  PythonToken NextToken();
88  const PythonToken& CurrentToken() const;
89  void SetInString(bool in_string);
90  bool InString() const { return in_string_; }
91  const QString& GetDelim() const { return string_delim_; }
92 private:
93  void EatWhities();
94  QString GetSubString(const Range& range);
95  PythonToken GetIdentifierSeq();
96  PythonToken GetNumberToken();
97  PythonToken GetKeywordToken();
98  PythonToken GetOperatorToken();
99  PythonToken GetGroupingToken();
100  PythonToken GetStringLiteral();
101  PythonToken GetStringDelim();
102  QString command_;
103  int current_pos_;
104  bool in_string_;
105  QString string_delim_;
106  PythonToken current_token_;
107 };
108 
109 
110 }}
111 
112 #endif